From 338b52911ff24b26e409d1fc911a2d134e9d1f15 Mon Sep 17 00:00:00 2001 From: GEXCE Date: Wed, 8 Nov 2017 17:38:10 +0100 Subject: [PATCH] Upload working --- .../resources/application.properties | 2 +- .../com/bayer/edam/migration/Definitions.java | 2 + .../edam/migration/MamMigrationSelective.java | 75 +++++++++++++++++-- .../bayer/edam/migration/UploadReference.java | 9 +-- .../edam/migration/adam/UploadFileRecord.java | 55 ++++++++++++++ 5 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/adam/UploadFileRecord.java diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index 77fb5e5..e783bf8 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -2,4 +2,4 @@ logging.level.com.bayer=DEBUG #logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application-TUE.log +logging.file=application-WED.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java index cadd40e..b30e5ce 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java @@ -187,6 +187,7 @@ public class Definitions { fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef"); fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a"); fieldIDs.put("byr_ah_photographer", "4499ba20-5828-4b98-9e07-a731010bbcef"); + fieldIDs.put("byr_ah_releaseDate", "7af57306-002a-4fc4-9273-a78300bdb5bb"); // Users users.put("32300", "02bee145-2a37-4f49-acba-a74600bda888"); // marion.schmidtmeier@bayer.com @@ -337,6 +338,7 @@ public class Definitions { // fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "e7ded420-ac62-49c5-8506-a73e00dbab8d"); // fieldIDs.put("byr_ah_licenseComments", "f951aaad-2a58-4065-adb8-a6db0116f9f8"); // fieldIDs.put("byr_ah_photographer", "0ae51b77-388e-4d07-8f6a-a73e00dba54e"); +// fieldIDs.put("byr_ah_releaseDate", "820c46c1-d98e-436e-8aee-a76800863984"); // // // Users // users.put("32421", "c57444c4-7149-4ee2-8f5d-a7290078047f"); // claudia.staubes-dunkel@bayer.com diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 243a8bd..4073a3e 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -16,6 +16,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -44,6 +45,7 @@ import com.bayer.edam.migration.adam.ClassificationList; import com.bayer.edam.migration.adam.EntityId; import com.bayer.edam.migration.adam.Token; import com.bayer.edam.migration.adam.UploadClassification; +import com.bayer.edam.migration.adam.UploadFileRecord; import com.bayer.edam.migration.adam.UploadRecord; import com.bayer.edam.migration.adam.UploadRecord.WriteClassification; import com.bayer.edam.migration.adam.UploadResult; @@ -163,6 +165,14 @@ public class MamMigrationSelective implements CommandLineRunner { postToAdam(entry); } + while (openUploads.size() > 0) { + log.debug("Sleeping..."); + TimeUnit.SECONDS.sleep(10); // XXX + + log.debug("Checking back on open uploads"); + attachOpenUploads(); + } + invalidateAdamToken(); } @@ -361,7 +371,7 @@ public class MamMigrationSelective implements CommandLineRunner { } if (entry.creationDate != null) - record.addField("byr_ah_deliveryDate", isoDate.format(entry.creationDate)); + record.addField("byr_ah_releaseDate", isoDate.format(entry.creationDate)); if (entry.metaData.get("photographer") != null) record.addField("byr_ah_photographer", entry.metaData.get("photographer")); @@ -456,10 +466,9 @@ public class MamMigrationSelective implements CommandLineRunner { URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); log.info("RESULT Entry " + entry.id + " migrated to record " + location); - // XXX uploadFilesForEntry(entry, location); - // TODO process file upload errors + // TODO process file upload errors(?) log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); } @@ -526,6 +535,7 @@ public class MamMigrationSelective implements CommandLineRunner { } // Instead of downloading from MAM, take the pre-downloaded files from the harddrive + boolean isMaster = true; for (File file : filesForUpload) { if (file.exists()) { if (file.length() < 4000000000L) { @@ -538,13 +548,22 @@ public class MamMigrationSelective implements CommandLineRunner { new HttpEntity>(parameters, adamUploadHeaders), UploadResult.class).getBody(); if (result.token != null) { - // TODO - - log.debug("Token: " + result.token); + log.debug("Storing upload token " + result.token); + UploadReference reference = new UploadReference(); + reference.targetRecord = recordLocation; +// reference.isMaster = isMaster; + reference.token = result.token; + reference.filename = file.getName(); + openUploads.add(reference); } else if (result.uri != null) { log.debug("Storing upload reference URI " + result.uri); - openUploads.add(new UploadReference(recordLocation, result.uri)); + UploadReference reference = new UploadReference(); + reference.targetRecord = recordLocation; +// reference.isMaster = isMaster; + reference.uri = result.uri; + reference.filename = file.getName(); + openUploads.add(reference); } else { log.error("No token/URI after upload!"); @@ -560,9 +579,49 @@ public class MamMigrationSelective implements CommandLineRunner { else { log.error("Could not find file on disk " + file.getAbsolutePath()); } + + isMaster = false; } - + // attach uploaded files to records where possible + attachOpenUploads(); + + } + + private void attachOpenUploads() { + List processedReferences = new ArrayList(); + + for (UploadReference reference : openUploads) { + if (reference.token != null) { + UploadFileRecord fileRecord = new UploadFileRecord(); + fileRecord.addFile(reference.token, reference.filename); + + log.debug("Attach uploaded file to " + reference.targetRecord); + HttpEntity httpEntity = new HttpEntity(fileRecord, adamHeaders); + restTemplate.exchange(Definitions.adamHost + reference.targetRecord, HttpMethod.PUT, httpEntity, UploadFileRecord.class); + + processedReferences.add(reference); + } + else { + log.debug("Checking status of upload " + reference.filename + " at " + reference.uri); + UploadResult result = restTemplate.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class).getBody(); + log.debug("Result: " + result); + + if (result.token != null) { + log.debug("Storing upload token " + result.token); + reference.uri = null; + reference.token = result.token; + } + else if (result.status != null) { + if (result.status.equals("Error")) { + log.error("System could not process file " + reference.filename + " for " + reference.targetRecord); + processedReferences.add(reference); + } + } + } + } + + openUploads.removeAll(processedReferences); } private File getFileForEntry(Download download, Long id) throws IOException { diff --git a/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java b/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java index 6a5c830..4380259 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java @@ -5,11 +5,10 @@ import java.net.URI; public class UploadReference { URI targetRecord; - String fileURI; + String uri; + String token; - public UploadReference(URI record, String fileURI) { - this.targetRecord = record; - this.fileURI = fileURI; - } + String filename; +// String isMaster; } diff --git a/AH MAM Migration/src/com/bayer/edam/migration/adam/UploadFileRecord.java b/AH MAM Migration/src/com/bayer/edam/migration/adam/UploadFileRecord.java new file mode 100644 index 0000000..ddee62d --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/adam/UploadFileRecord.java @@ -0,0 +1,55 @@ +package com.bayer.edam.migration.adam; + +import java.util.ArrayList; +import java.util.List; + +public class UploadFileRecord { + + public static class WriteFile { + public WriteFile(String id, String filename) { + this.id = id; + this.filename = filename; + } + + public String id; + public String filename; + } + + + public static class Versions { + public Versions(WriteFile file) { + this.addOrUpdate.add(file); + } + + public List addOrUpdate = new ArrayList(); + } + + + public static class FileContent { + public FileContent(Versions versions) { + this.versions = versions; + } + + public Versions versions; + } + + public static class Files { +// public String master; + public List addOrUpdate = new ArrayList(); + } + + public UploadFileRecord(String masterToken, String masterFilename) { +// files.master = masterToken; + addFile(masterToken, masterFilename); + } + + public UploadFileRecord() { + } + + public String contentType = "Record"; + public Files files = new Files(); + + public void addFile(String token, String filename) { + files.addOrUpdate.add(new FileContent(new Versions(new WriteFile(token, filename)))); + } +}