tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uploads

Browse Source
This commit is contained in:
gexce
2017-11-07 16:07:27 +01:00
parent 79b71a2e7e
commit 4fdf6182b3
7 changed files with 68 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/notes.txt
+9 -9
View File
@@ -11,22 +11,22 @@
Advertising Media FAP Advertising Media FAP
MA-ID 00070142 OK MA-ID 00070142 OK
MA-ID 00070361 MA-ID 00070361 OK
MA-ID 00063924 MA-ID 00063924 OK
Images Images
MA-ID 00071263 OK MA-ID 00071263 OK
MA-ID 00063201 MA-ID 00063201 OK
MA-ID 00056942 MA-ID 00056942 OK
MA-ID 00057641 MA-ID 00057641 OK
Videos (mit allen Downloadmöglichkeiten) Videos (mit allen Downloadmöglichkeiten)
MA-ID 00070745 MA-ID 00070745 OK
MA-ID 00070440 MA-ID 00070440 OK
MA-ID 00070340 MA-ID 00070340 OK
Guidelines Guidelines
MA-ID 00070288 MA-ID 00070288 OK
MA-ID 00071725 OK MA-ID 00071725 OK
- PROD: Problem mit Campaigns: - PROD: Problem mit Campaigns:
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO #logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application-WEEKEND.log logging.file=application-TUE.log
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+4
View File
@@ -186,7 +186,9 @@ public class Definitions {
fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c"); fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c");
fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef"); fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a"); fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a");
fieldIDs.put("byr_ah_photographer", "4499ba20-5828-4b98-9e07-a731010bbcef");
// Users
users.put("32300", "02bee145-2a37-4f49-acba-a74600bda888"); // marion.schmidtmeier@bayer.com users.put("32300", "02bee145-2a37-4f49-acba-a74600bda888"); // marion.schmidtmeier@bayer.com
users.put("32421", "c68e1a72-f218-48f0-ac4d-a72900796cc1"); // claudia.staubes-dunkel@bayer.com users.put("32421", "c68e1a72-f218-48f0-ac4d-a72900796cc1"); // claudia.staubes-dunkel@bayer.com
users.put("32422", "ef573ad7-aa7f-44cd-9c5f-a743007ecfb6"); // roswitha.wolff@bayer.com users.put("32422", "ef573ad7-aa7f-44cd-9c5f-a743007ecfb6"); // roswitha.wolff@bayer.com
@@ -334,7 +336,9 @@ public class Definitions {
// fieldIDs.put("byr_ah_licenseHolderList", "fcc8bd4e-cf1d-4e88-b3dc-a73e00dba7d6"); // fieldIDs.put("byr_ah_licenseHolderList", "fcc8bd4e-cf1d-4e88-b3dc-a73e00dba7d6");
// fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "e7ded420-ac62-49c5-8506-a73e00dbab8d"); // fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "e7ded420-ac62-49c5-8506-a73e00dbab8d");
// fieldIDs.put("byr_ah_licenseComments", "f951aaad-2a58-4065-adb8-a6db0116f9f8"); // fieldIDs.put("byr_ah_licenseComments", "f951aaad-2a58-4065-adb8-a6db0116f9f8");
// fieldIDs.put("byr_ah_photographer", "0ae51b77-388e-4d07-8f6a-a73e00dba54e");
// //
// // Users
// users.put("32421", "c57444c4-7149-4ee2-8f5d-a7290078047f"); // claudia.staubes-dunkel@bayer.com // users.put("32421", "c57444c4-7149-4ee2-8f5d-a7290078047f"); // claudia.staubes-dunkel@bayer.com
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+3 -3
View File
@@ -40,9 +40,9 @@ public class EntryDownloader implements CommandLineRunner {
/** /**
* If true, files are downloaded. If false, only the IDs are written into the outputfile. * If true, files are downloaded. If false, only the IDs are written into the outputfile.
*/ */
private final boolean downloadFiles = false; private final boolean downloadFiles = true;
File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-10-25.txt"); File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-07.txt");
@Value("${mam.auth}") @Value("${mam.auth}")
@@ -96,7 +96,6 @@ public class EntryDownloader implements CommandLineRunner {
for (int i = 0; i < entrylist.entries.length; i++) { for (int i = 0; i < entrylist.entries.length; i++) {
if (!downloadFiles) { if (!downloadFiles) {
// only create a list with all IDs // only create a list with all IDs
try { try {
@@ -108,6 +107,7 @@ public class EntryDownloader implements CommandLineRunner {
else { else {
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId); Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
File targetDir = new File("E:\\" + entryId); File targetDir = new File("E:\\" + entryId);
if (targetDir.exists()) { if (targetDir.exists()) {
log.info("Skip " + entryId + ", target directory already present"); log.info("Skip " + entryId + ", target directory already present");
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+20 -18
View File
@@ -2,7 +2,6 @@ package com.bayer.edam.migration;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
@@ -43,6 +42,7 @@ import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification; import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadRecord; import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification; import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.adam.UploadResult;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor; import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.CategoryTree; import com.bayer.edam.migration.mam.CategoryTree;
import com.bayer.edam.migration.mam.CategoryTree.Node; import com.bayer.edam.migration.mam.CategoryTree.Node;
@@ -57,7 +57,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class); private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 72048, 60984 /*48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984 private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 71263 /*72048, 60984 /*48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
@@ -90,7 +90,7 @@ public class MamMigrationSelective implements CommandLineRunner {
public void run(String... strings) throws Exception { public void run(String... strings) throws Exception {
// XXX // XXX
readEntryIDsFromFile(); // readEntryIDsFromFile();
log.info("Write log headers for " + entryLog.getAbsolutePath()); log.info("Write log headers for " + entryLog.getAbsolutePath());
FileUtils.writeStringToFile(entryLog, "entryId" + fieldSeparator + "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8")); FileUtils.writeStringToFile(entryLog, "entryId" + fieldSeparator + "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
@@ -127,8 +127,8 @@ public class MamMigrationSelective implements CommandLineRunner {
// XXX // XXX
// if needed start from a certain ID // if needed start from a certain ID
if (id >= 32935) // if (id > 32348)
continue; // continue;
log.info("GET entry " + id); log.info("GET entry " + id);
@@ -358,6 +358,9 @@ public class MamMigrationSelective implements CommandLineRunner {
if (entry.creationDate != null) if (entry.creationDate != null)
record.addField("byr_ah_deliveryDate", isoDate.format(entry.creationDate)); record.addField("byr_ah_deliveryDate", isoDate.format(entry.creationDate));
if (entry.metaData.get("photographer") != null)
record.addField("byr_ah_photographer", entry.metaData.get("photographer"));
// Project lead // Project lead
if (entry.user != null && entry.user.get("id") != null) { if (entry.user != null && entry.user.get("id") != null) {
if (Definitions.users.get(entry.user.get("id")) != null) { if (Definitions.users.get(entry.user.get("id")) != null) {
@@ -437,16 +440,16 @@ public class MamMigrationSelective implements CommandLineRunner {
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders); HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try { try {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); // URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location); // log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX // XXX
// uploadFilesForEntry(entry); uploadFilesForEntry(entry);
// TODO process file upload errors // TODO process file upload errors
FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator, // FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true); // Charset.forName("utf-8"), true);
} }
catch (HttpStatusCodeException hsce) { catch (HttpStatusCodeException hsce) {
log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString()); log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -455,7 +458,7 @@ public class MamMigrationSelective implements CommandLineRunner {
} }
private void uploadFilesForEntry(Entry entry) { private void uploadFilesForEntry(Entry entry) throws Exception {
List<Download> downloadSelection = new ArrayList<Download>(); List<Download> downloadSelection = new ArrayList<Download>();
@@ -504,18 +507,17 @@ public class MamMigrationSelective implements CommandLineRunner {
if (uploadFile.exists()) { if (uploadFile.exists()) {
if (uploadFile.length() < 4000000000L) { if (uploadFile.length() < 4000000000L) {
log.debug("filesize " + uploadFile.length() + " < 4 GB"); log.info("Uploading file " + filepath + ", size " + uploadFile.length());
log.info("Uploading file " + filepath);
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>(); MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(uploadFile)); parameters.add("file", new FileSystemResource(uploadFile));
String result = restTemplate.postForObject(Definitions.adamHost + "/uploads", UploadResult result = restTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class); new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
// String answer = restTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST, if (result.token != null) {
// new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class).getBody(); log.debug("Token: " + result.token);
}
log.info("Upload result: " + result); log.info("Upload result: " + result);
AH MAM Migration/src/com/bayer/edam/migration/adam/UploadResult.java
+21
View File
@@ -0,0 +1,21 @@
package com.bayer.edam.migration.adam;
public class UploadResult {
public String token;
public String uri;
public String status;
@Override
public String toString() {
if (token != null)
return "{\"token\":\"" + token + "\"}";
else if (uri != null)
return "{\"uri\":\"" + uri + "\"}";
else if (status != null)
return "{\"status\":\"" + status + "\"}";
return null;
}
}
AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java
+10 -1
View File
@@ -3,6 +3,7 @@ package com.bayer.edam.migration.interceptors;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.Arrays;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -26,11 +27,19 @@ public class LoggingRequestInterceptor implements ClientHttpRequestInterceptor {
} }
private void traceRequest(HttpRequest request, byte[] body) throws IOException { private void traceRequest(HttpRequest request, byte[] body) throws IOException {
String bodyRepresentation = null;
if (body.length > 1 && body[0] == '-' && body[1] == '-')
bodyRepresentation = new String(Arrays.copyOfRange(body, 0, 32), "UTF-8");
else
bodyRepresentation = new String(body, "UTF-8");
log.debug("===================request begin==================="); log.debug("===================request begin===================");
log.debug("URI : {}", request.getURI()); log.debug("URI : {}", request.getURI());
log.debug("Method : {}", request.getMethod()); log.debug("Method : {}", request.getMethod());
log.debug("Headers : {}", request.getHeaders()); log.debug("Headers : {}", request.getHeaders());
log.debug("Request body: {}", new String(body, "UTF-8")); log.debug("Request body: {}", bodyRepresentation);
log.debug("====================request end===================="); log.debug("====================request end====================");
} }