tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

File Upload Error

Browse Source
This commit is contained in:
gexce
2017-11-02 11:23:16 +01:00
parent 516a4a4b46
commit 1585c9e29a
6 changed files with 115 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+109 -19
View File
@@ -8,6 +8,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -21,6 +22,7 @@ import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
@@ -28,6 +30,8 @@ import org.springframework.http.ResponseEntity;
import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestTemplate;
@@ -39,9 +43,11 @@ import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.CategoryTree;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.CategoryTree.Node;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
@SpringBootApplication
@@ -52,15 +58,10 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String adamHost = "https://edam-q.bayer.com:2015";
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 63701, 63711, 43182 )); // 72048, 60984
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
static final File entryLog = new File("entry.log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@Value("${mam.auth}")
private String mamAuth;
@@ -69,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private String adamAuth;
RestTemplate restTemplate;
HttpHeaders adamHeaders, mamHeaders;
HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
HttpEntity<String> adamHeadersEntity;
private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
@@ -80,6 +81,12 @@ public class MamMigrationSelective implements CommandLineRunner {
private List<Campaign> campaigns = new LinkedList<Campaign>();
private CategoryTree mamCategoryTree;
static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@Override
public void run(String... strings) throws Exception {
@@ -99,6 +106,9 @@ public class MamMigrationSelective implements CommandLineRunner {
adamHeaders = createAdamHeaders();
adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
adamUploadHeaders = createAdamHeaders();
adamUploadHeaders.set("Content-Type", "multipart/form-data");
adamUploadHeaders.set("Accept", "*/*");
mamHeaders = createMamHeaders();
log.info("Read campaigns");
@@ -118,7 +128,7 @@ public class MamMigrationSelective implements CommandLineRunner {
// XXX
// if needed start from a certain ID
// if (id > 71731)
// if (id > 32954)
// continue;
@@ -238,11 +248,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
if (p.startsWith("MediaType.")) {
record.removeClassification("MediaType.");
}
if (p.startsWith("Motive.")) {
record.removeClassification("Motive.");
record.removeClassificationsStartingWith("MediaType.");
}
record.addClassification(p);
@@ -431,11 +437,17 @@ public class MamMigrationSelective implements CommandLineRunner {
// Create record
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
// URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
// log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX
uploadFilesForEntry(entry);
// TODO process file upload errors
// FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
// Charset.forName("utf-8"), true);
}
catch (HttpStatusCodeException hsce) {
log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -443,6 +455,84 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
private void uploadFilesForEntry(Entry entry) {
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// Clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
String filepath = "E:\\" + entry.id + "\\" + filenameToUse;
File uploadFile = new File(filepath);
if (uploadFile.exists()) {
if (uploadFile.length() < 4000000000L) {
log.debug("filesize " + uploadFile.length() + " < 4 GB");
log.info("Uploading file " + filepath);
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(uploadFile));
String result = restTemplate.postForObject(adamHost + "/uploads",
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class);
// String answer = restTemplate.exchange(adamHost + "/uploads", HttpMethod.POST,
// new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class).getBody();
log.info("Upload result: " + result);
}
else {
log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!");
}
}
else {
log.error("Could not find file on disk " + filepath);
}
}
}
private void readEntryIDsFromFile() throws IOException {
File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());
AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java
+1 -1
View File
@@ -22,10 +22,10 @@ import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.LoggingRequestInterceptor;
import com.bayer.edam.migration.adam.ReadRecord;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.Entry;
public class MediaAssistantMigrationSimple {
AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java → AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java
+1 -1
View File
@@ -1,4 +1,4 @@
package com.bayer.edam.migration;
package com.bayer.edam.migration.interceptors;
import java.io.BufferedReader;
import java.io.IOException;