tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

File Upload Error

Browse Source
This commit is contained in:
gexce
2017-11-02 11:23:16 +01:00
parent 516a4a4b46
commit 1585c9e29a
6 changed files with 115 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv
+2 -2
View File
@@ -497,8 +497,8 @@ keyword;;;;;;;Packaging Images;;;
classify;BusinessUnit.FAP,Brand.FAP.Others;;;;FAP Brand Design;;;;;
skip;;;;;;Farm Animal Products;;;;
skip;;;;;;Farmhygiene Products;;;;
classify;;;;;Packaging;;;;;
classify;;;;;;Absolut;;;;
skip;;;;;Packaging;;;;;
skip;;;;;;Absolut;;;;
classify;Brand.Advantage;;;;;Advantage;;;;
classify;Brand.Advantix;;;;;Advantix;;;;
skip;;;;;Training Tools MA;;;;;
1 Status Business Unit Brand
497 skip Farmhygiene Products
498 classify skip Packaging
499 classify skip Absolut
500 classify Brand.Advantage Advantage
501 classify Brand.Advantix Advantix
502 skip Training Tools MA
503 skip eDetailer Template
504 classify BusinessUnit.General,Brand.General Illustrations --> General
AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.xlsx
BIN
View File
Binary file not shown.
AH MAM Migration/resources/application.properties
+2 -1
View File
@@ -1,4 +1,5 @@
# TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application.log
logging.file=test.log
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+108 -18
View File
@@ -8,6 +8,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -21,6 +22,7 @@ import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
@@ -28,6 +30,8 @@ import org.springframework.http.ResponseEntity;
import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestTemplate;
@@ -39,9 +43,11 @@ import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.CategoryTree;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.CategoryTree.Node;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
@SpringBootApplication
@@ -52,16 +58,11 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String adamHost = "https://edam-q.bayer.com:2015";
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 63701, 63711, 43182 )); // 72048, 60984
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
static final File entryLog = new File("entry.log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@Value("${mam.auth}")
private String mamAuth;
@@ -69,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private String adamAuth;
RestTemplate restTemplate;
HttpHeaders adamHeaders, mamHeaders;
HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
HttpEntity<String> adamHeadersEntity;
private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
@@ -80,6 +81,12 @@ public class MamMigrationSelective implements CommandLineRunner {
private List<Campaign> campaigns = new LinkedList<Campaign>();
private CategoryTree mamCategoryTree;
static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@Override
public void run(String... strings) throws Exception {
@@ -99,6 +106,9 @@ public class MamMigrationSelective implements CommandLineRunner {
adamHeaders = createAdamHeaders();
adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
adamUploadHeaders = createAdamHeaders();
adamUploadHeaders.set("Content-Type", "multipart/form-data");
adamUploadHeaders.set("Accept", "*/*");
mamHeaders = createMamHeaders();
log.info("Read campaigns");
@@ -118,7 +128,7 @@ public class MamMigrationSelective implements CommandLineRunner {
// XXX
// if needed start from a certain ID
// if (id > 71731)
// if (id > 32954)
// continue;
@@ -238,11 +248,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
if (p.startsWith("MediaType.")) {
record.removeClassification("MediaType.");
}
if (p.startsWith("Motive.")) {
record.removeClassification("Motive.");
record.removeClassificationsStartingWith("MediaType.");
}
record.addClassification(p);
@@ -431,11 +437,17 @@ public class MamMigrationSelective implements CommandLineRunner {
// Create record
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
// URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
// log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX
uploadFilesForEntry(entry);
// TODO process file upload errors
// FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
// Charset.forName("utf-8"), true);
}
catch (HttpStatusCodeException hsce) {
log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -443,6 +455,84 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
private void uploadFilesForEntry(Entry entry) {
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// Clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
String filepath = "E:\\" + entry.id + "\\" + filenameToUse;
File uploadFile = new File(filepath);
if (uploadFile.exists()) {
if (uploadFile.length() < 4000000000L) {
log.debug("filesize " + uploadFile.length() + " < 4 GB");
log.info("Uploading file " + filepath);
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(uploadFile));
String result = restTemplate.postForObject(adamHost + "/uploads",
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class);
// String answer = restTemplate.exchange(adamHost + "/uploads", HttpMethod.POST,
// new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class).getBody();
log.info("Upload result: " + result);
}
else {
log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!");
}
}
else {
log.error("Could not find file on disk " + filepath);
}
}
}
private void readEntryIDsFromFile() throws IOException {
File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());
AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java
+1 -1
View File
@@ -22,10 +22,10 @@ import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.LoggingRequestInterceptor;
import com.bayer.edam.migration.adam.ReadRecord;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.Entry;
public class MediaAssistantMigrationSimple {
AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java → AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java
+1 -1
View File
@@ -1,4 +1,4 @@
package com.bayer.edam.migration;
package com.bayer.edam.migration.interceptors;
import java.io.BufferedReader;
import java.io.IOException;