File Upload Error

AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv

@@ -497,8 +497,8 @@ keyword;;;;;;;Packaging Images;;;
 classify;BusinessUnit.FAP,Brand.FAP.Others;;;;FAP Brand Design;;;;;
 skip;;;;;;Farm Animal Products;;;;
 skip;;;;;;Farmhygiene Products;;;;
-classify;;;;;Packaging;;;;;
+skip;;;;;Packaging;;;;;
-classify;;;;;;Absolut;;;;
+skip;;;;;;Absolut;;;;
 classify;Brand.Advantage;;;;;Advantage;;;;
 classify;Brand.Advantix;;;;;Advantix;;;;
 skip;;;;;Training Tools MA;;;;;

AH MAM Migration/resources/application.properties

@@ -1,4 +1,5 @@
 # TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
 logging.level.com.bayer=DEBUG
+#logging.level.com.bayer.edam.migration.interceptors=INFO
 logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
-logging.file=application.log
+logging.file=test.log

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -8,6 +8,7 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -21,6 +22,7 @@ import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.PropertySource;
+import org.springframework.core.io.FileSystemResource;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
@@ -28,6 +30,8 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.http.client.BufferingClientHttpRequestFactory;
 import org.springframework.http.client.ClientHttpRequestInterceptor;
 import org.springframework.http.client.SimpleClientHttpRequestFactory;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
 import org.springframework.web.client.HttpStatusCodeException;
 import org.springframework.web.client.RestTemplate;
 
@@ -39,9 +43,11 @@ import com.bayer.edam.migration.adam.Token;
 import com.bayer.edam.migration.adam.UploadClassification;
 import com.bayer.edam.migration.adam.UploadRecord;
 import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
+import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
 import com.bayer.edam.migration.mam.CategoryTree;
-import com.bayer.edam.migration.mam.Entry;
 import com.bayer.edam.migration.mam.CategoryTree.Node;
+import com.bayer.edam.migration.mam.Entry;
+import com.bayer.edam.migration.mam.Entry.Download;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 @SpringBootApplication
@@ -52,16 +58,11 @@ public class MamMigrationSelective implements CommandLineRunner {
 	
 	static final String adamHost = "https://edam-q.bayer.com:2015";
 	static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
-	private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 63701, 63711, 43182 )); // 72048, 60984
+	private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
 	
 	static final String categoryJson = "resources/2017-10-19_categories.json";
 	static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
 
-	static final File entryLog = new File("entry.log");
-	
-	static final String fieldSeparator = "|~|";
-	static final String lineSeparator = "\r\n";
-
 	@Value("${mam.auth}")
 	private String mamAuth;
 
@@ -69,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private String adamAuth;
 
 	RestTemplate restTemplate;
-	HttpHeaders adamHeaders, mamHeaders;
+	HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
 	HttpEntity<String> adamHeadersEntity;
 
 	private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
@@ -80,6 +81,12 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private List<Campaign> campaigns = new LinkedList<Campaign>();
 	private CategoryTree mamCategoryTree;
 	
+	
+	static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
+	static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log");
+	static final String fieldSeparator = "|~|";
+	static final String lineSeparator = "\r\n";
+	
 	@Override
 	public void run(String... strings) throws Exception {
 		
@@ -99,6 +106,9 @@ public class MamMigrationSelective implements CommandLineRunner {
 
 		adamHeaders = createAdamHeaders();
 		adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
+		adamUploadHeaders = createAdamHeaders();
+		adamUploadHeaders.set("Content-Type", "multipart/form-data");
+		adamUploadHeaders.set("Accept", "*/*");
 		mamHeaders = createMamHeaders();
 		
 		log.info("Read campaigns");
@@ -118,7 +128,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 			
 			// XXX
 			// if needed start from a certain ID
-//			if (id > 71731)
+//			if (id > 32954)
 //				continue;
 			
 			
@@ -238,11 +248,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 						}
 						
 						if (p.startsWith("MediaType.")) {
-							record.removeClassification("MediaType.");
+							record.removeClassificationsStartingWith("MediaType.");
-						}
-						
-						if (p.startsWith("Motive.")) {
-							record.removeClassification("Motive.");
 						}
 						
 						record.addClassification(p);
@@ -431,11 +437,17 @@ public class MamMigrationSelective implements CommandLineRunner {
 		// Create record
 		HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
 		try {
-			URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
-			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
 			
-			FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
+//			URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
-					Charset.forName("utf-8"), true);
+//			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
+			
+			// XXX
+			uploadFilesForEntry(entry);
+			
+			// TODO process file upload errors
+			
+//			FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
+//					Charset.forName("utf-8"), true);
 		}
 		catch (HttpStatusCodeException hsce) {
 			log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -443,6 +455,84 @@ public class MamMigrationSelective implements CommandLineRunner {
 		}
 	}
 	
+	
+	private void uploadFilesForEntry(Entry entry) {
+		
+		List<Download> downloadSelection = new ArrayList<Download>();
+		
+		// Select download: Download original only. If original is not available, download largest file
+		Download selectedDownload = null;
+		for (Download download : entry.downloadList) {
+			if (download.entryFileVariation.equals("ORIGINAL")) {
+				selectedDownload = download;
+				break;
+			}
+			
+			if (selectedDownload == null) {
+				selectedDownload = download;
+			}
+			else if (selectedDownload.size < download.size) {
+				selectedDownload = download;
+			}
+		}
+		
+		if (selectedDownload != null)
+			downloadSelection.add(selectedDownload);
+		
+		for (Download download : entry.downloadListPublicFolder) {
+			downloadSelection.add(download);
+		}
+		
+		// Clean download selection
+		HashSet<String> hrefs = new HashSet<String>();
+		List<Download> cleanedDownloads = new ArrayList<Download>();
+		for (Download download : downloadSelection) {
+			if (hrefs.add(download.href)) {
+				cleanedDownloads.add(download);
+			}
+		}
+		
+		// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
+		for (Download download : cleanedDownloads) {
+			String filenameToUse;
+			if (download.fileNameOnDisc != null)
+				filenameToUse = download.fileNameOnDisc;
+			else
+				filenameToUse = download.fileName;
+			
+			String filepath = "E:\\" + entry.id + "\\" + filenameToUse;
+			File uploadFile = new File(filepath);
+			
+			if (uploadFile.exists()) {
+				if (uploadFile.length() < 4000000000L) {
+					log.debug("filesize " + uploadFile.length() + " < 4 GB");
+					
+					log.info("Uploading file " + filepath);
+					
+					MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
+					parameters.add("file", new FileSystemResource(uploadFile));
+
+					String result = restTemplate.postForObject(adamHost + "/uploads",
+							new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class);
+					
+//					String answer = restTemplate.exchange(adamHost + "/uploads", HttpMethod.POST,
+//							new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), String.class).getBody();
+					
+					log.info("Upload result: " + result);
+					
+				}
+				else {
+					log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!");
+				}
+			}
+			else {
+				log.error("Could not find file on disk " + filepath);
+			}
+		}
+
+		
+	}
+
 	private void readEntryIDsFromFile() throws IOException {
 		File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
 		log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());

AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java

@@ -22,10 +22,10 @@ import org.springframework.http.client.SimpleClientHttpRequestFactory;
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.client.RestTemplate;
 
-import com.bayer.edam.migration.LoggingRequestInterceptor;
 import com.bayer.edam.migration.adam.ReadRecord;
 import com.bayer.edam.migration.adam.Token;
 import com.bayer.edam.migration.adam.UploadRecord;
+import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
 import com.bayer.edam.migration.mam.Entry;
 
 public class MediaAssistantMigrationSimple {

AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java

@@ -1,4 +1,4 @@
-package com.bayer.edam.migration;
+package com.bayer.edam.migration.interceptors;
 
 import java.io.BufferedReader;
 import java.io.IOException;