tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload working

Browse Source
This commit is contained in:
GEXCE
2017-11-08 17:38:10 +01:00
parent bb74e36db7
commit 338b52911f
5 changed files with 129 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application-TUE.log
logging.file=application-WED.log
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+2
View File
@@ -187,6 +187,7 @@ public class Definitions {
fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a");
fieldIDs.put("byr_ah_photographer", "4499ba20-5828-4b98-9e07-a731010bbcef");
fieldIDs.put("byr_ah_releaseDate", "7af57306-002a-4fc4-9273-a78300bdb5bb");
// Users
users.put("32300", "02bee145-2a37-4f49-acba-a74600bda888"); // marion.schmidtmeier@bayer.com
@@ -337,6 +338,7 @@ public class Definitions {
// fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "e7ded420-ac62-49c5-8506-a73e00dbab8d");
// fieldIDs.put("byr_ah_licenseComments", "f951aaad-2a58-4065-adb8-a6db0116f9f8");
// fieldIDs.put("byr_ah_photographer", "0ae51b77-388e-4d07-8f6a-a73e00dba54e");
// fieldIDs.put("byr_ah_releaseDate", "820c46c1-d98e-436e-8aee-a76800863984");
//
// // Users
// users.put("32421", "c57444c4-7149-4ee2-8f5d-a7290078047f"); // claudia.staubes-dunkel@bayer.com
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+67 -8
View File
@@ -16,6 +16,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
@@ -44,6 +45,7 @@ import com.bayer.edam.migration.adam.ClassificationList;
import com.bayer.edam.migration.adam.EntityId;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadFileRecord;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.adam.UploadResult;
@@ -163,6 +165,14 @@ public class MamMigrationSelective implements CommandLineRunner {
postToAdam(entry);
}
while (openUploads.size() > 0) {
log.debug("Sleeping...");
TimeUnit.SECONDS.sleep(10); // XXX
log.debug("Checking back on open uploads");
attachOpenUploads();
}
invalidateAdamToken();
}
@@ -361,7 +371,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
if (entry.creationDate != null)
record.addField("byr_ah_deliveryDate", isoDate.format(entry.creationDate));
record.addField("byr_ah_releaseDate", isoDate.format(entry.creationDate));
if (entry.metaData.get("photographer") != null)
record.addField("byr_ah_photographer", entry.metaData.get("photographer"));
@@ -456,10 +466,9 @@ public class MamMigrationSelective implements CommandLineRunner {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX
uploadFilesForEntry(entry, location);
// TODO process file upload errors
// TODO process file upload errors(?)
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
}
@@ -526,6 +535,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
boolean isMaster = true;
for (File file : filesForUpload) {
if (file.exists()) {
if (file.length() < 4000000000L) {
@@ -538,13 +548,22 @@ public class MamMigrationSelective implements CommandLineRunner {
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
if (result.token != null) {
// TODO
log.debug("Token: " + result.token);
log.debug("Storing upload token " + result.token);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
// reference.isMaster = isMaster;
reference.token = result.token;
reference.filename = file.getName();
openUploads.add(reference);
}
else if (result.uri != null) {
log.debug("Storing upload reference URI " + result.uri);
openUploads.add(new UploadReference(recordLocation, result.uri));
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
// reference.isMaster = isMaster;
reference.uri = result.uri;
reference.filename = file.getName();
openUploads.add(reference);
}
else {
log.error("No token/URI after upload!");
@@ -560,9 +579,49 @@ public class MamMigrationSelective implements CommandLineRunner {
else {
log.error("Could not find file on disk " + file.getAbsolutePath());
}
isMaster = false;
}
// attach uploaded files to records where possible
attachOpenUploads();
}
private void attachOpenUploads() {
List<UploadReference> processedReferences = new ArrayList<UploadReference>();
for (UploadReference reference : openUploads) {
if (reference.token != null) {
UploadFileRecord fileRecord = new UploadFileRecord();
fileRecord.addFile(reference.token, reference.filename);
log.debug("Attach uploaded file to " + reference.targetRecord);
HttpEntity<UploadFileRecord> httpEntity = new HttpEntity<UploadFileRecord>(fileRecord, adamHeaders);
restTemplate.exchange(Definitions.adamHost + reference.targetRecord, HttpMethod.PUT, httpEntity, UploadFileRecord.class);
processedReferences.add(reference);
}
else {
log.debug("Checking status of upload " + reference.filename + " at " + reference.uri);
UploadResult result = restTemplate.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class).getBody();
log.debug("Result: " + result);
if (result.token != null) {
log.debug("Storing upload token " + result.token);
reference.uri = null;
reference.token = result.token;
}
else if (result.status != null) {
if (result.status.equals("Error")) {
log.error("System could not process file " + reference.filename + " for " + reference.targetRecord);
processedReferences.add(reference);
}
}
}
}
openUploads.removeAll(processedReferences);
}
private File getFileForEntry(Download download, Long id) throws IOException {
AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java
+4 -5
View File
@@ -5,11 +5,10 @@ import java.net.URI;
public class UploadReference {
URI targetRecord;
String fileURI;
String uri;
String token;
public UploadReference(URI record, String fileURI) {
this.targetRecord = record;
this.fileURI = fileURI;
}
String filename;
// String isMaster;
}
AH MAM Migration/src/com/bayer/edam/migration/adam/UploadFileRecord.java
+55
View File
@@ -0,0 +1,55 @@
package com.bayer.edam.migration.adam;
import java.util.ArrayList;
import java.util.List;
public class UploadFileRecord {
public static class WriteFile {
public WriteFile(String id, String filename) {
this.id = id;
this.filename = filename;
}
public String id;
public String filename;
}
public static class Versions {
public Versions(WriteFile file) {
this.addOrUpdate.add(file);
}
public List<WriteFile> addOrUpdate = new ArrayList<WriteFile>();
}
public static class FileContent {
public FileContent(Versions versions) {
this.versions = versions;
}
public Versions versions;
}
public static class Files {
// public String master;
public List<FileContent> addOrUpdate = new ArrayList<FileContent>();
}
public UploadFileRecord(String masterToken, String masterFilename) {
// files.master = masterToken;
addFile(masterToken, masterFilename);
}
public UploadFileRecord() {
}
public String contentType = "Record";
public Files files = new Files();
public void addFile(String token, String filename) {
files.addOrUpdate.add(new FileContent(new Versions(new WriteFile(token, filename))));
}
}