diff --git a/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv b/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv index 9f75966..abbba06 100644 --- a/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv +++ b/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv @@ -497,8 +497,8 @@ keyword;;;;;;;Packaging Images;;; classify;BusinessUnit.FAP,Brand.FAP.Others;;;;FAP Brand Design;;;;; skip;;;;;;Farm Animal Products;;;; skip;;;;;;Farmhygiene Products;;;; -classify;;;;;Packaging;;;;; -classify;;;;;;Absolut;;;; +skip;;;;;Packaging;;;;; +skip;;;;;;Absolut;;;; classify;Brand.Advantage;;;;;Advantage;;;; classify;Brand.Advantix;;;;;Advantix;;;; skip;;;;;Training Tools MA;;;;; diff --git a/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.xlsx b/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.xlsx index 941ad23..5ecbf26 100644 Binary files a/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.xlsx and b/AH MAM Migration/resources/Media_Assistant_Categories_131017_EDITED_without_archive.xlsx differ diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index db58c14..d773997 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -1,4 +1,5 @@ # TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF logging.level.com.bayer=DEBUG +#logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application.log +logging.file=test.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 273d64f..ecc42d3 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -8,6 +8,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -21,6 +22,7 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.PropertySource; +import org.springframework.core.io.FileSystemResource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -28,6 +30,8 @@ import org.springframework.http.ResponseEntity; import org.springframework.http.client.BufferingClientHttpRequestFactory; import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestTemplate; @@ -39,9 +43,11 @@ import com.bayer.edam.migration.adam.Token; import com.bayer.edam.migration.adam.UploadClassification; import com.bayer.edam.migration.adam.UploadRecord; import com.bayer.edam.migration.adam.UploadRecord.WriteClassification; +import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor; import com.bayer.edam.migration.mam.CategoryTree; -import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.CategoryTree.Node; +import com.bayer.edam.migration.mam.Entry; +import com.bayer.edam.migration.mam.Entry.Download; import com.fasterxml.jackson.databind.ObjectMapper; @SpringBootApplication @@ -52,15 +58,10 @@ public class MamMigrationSelective implements CommandLineRunner { static final String adamHost = "https://edam-q.bayer.com:2015"; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; - private List entrySelection = new ArrayList(Arrays.asList( 63701, 63711, 43182 )); // 72048, 60984 + private List entrySelection = new ArrayList(Arrays.asList( 48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984 static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; - - static final File entryLog = new File("entry.log"); - - static final String fieldSeparator = "|~|"; - static final String lineSeparator = "\r\n"; @Value("${mam.auth}") private String mamAuth; @@ -69,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner { private String adamAuth; RestTemplate restTemplate; - HttpHeaders adamHeaders, mamHeaders; + HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders; HttpEntity adamHeadersEntity; private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); @@ -80,6 +81,12 @@ public class MamMigrationSelective implements CommandLineRunner { private List campaigns = new LinkedList(); private CategoryTree mamCategoryTree; + + static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss"); + static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log"); + static final String fieldSeparator = "|~|"; + static final String lineSeparator = "\r\n"; + @Override public void run(String... strings) throws Exception { @@ -99,6 +106,9 @@ public class MamMigrationSelective implements CommandLineRunner { adamHeaders = createAdamHeaders(); adamHeadersEntity = new HttpEntity("", adamHeaders); + adamUploadHeaders = createAdamHeaders(); + adamUploadHeaders.set("Content-Type", "multipart/form-data"); + adamUploadHeaders.set("Accept", "*/*"); mamHeaders = createMamHeaders(); log.info("Read campaigns"); @@ -118,7 +128,7 @@ public class MamMigrationSelective implements CommandLineRunner { // XXX // if needed start from a certain ID -// if (id > 71731) +// if (id > 32954) // continue; @@ -238,11 +248,7 @@ public class MamMigrationSelective implements CommandLineRunner { } if (p.startsWith("MediaType.")) { - record.removeClassification("MediaType."); - } - - if (p.startsWith("Motive.")) { - record.removeClassification("Motive."); + record.removeClassificationsStartingWith("MediaType."); } record.addClassification(p); @@ -431,11 +437,17 @@ public class MamMigrationSelective implements CommandLineRunner { // Create record HttpEntity httpEntity = new HttpEntity(record, adamHeaders); try { - URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class); - log.info("RESULT Entry " + entry.id + " migrated to record " + location); - - FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator, - Charset.forName("utf-8"), true); + +// URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class); +// log.info("RESULT Entry " + entry.id + " migrated to record " + location); + + // XXX + uploadFilesForEntry(entry); + + // TODO process file upload errors + +// FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator, +// Charset.forName("utf-8"), true); } catch (HttpStatusCodeException hsce) { log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString()); @@ -443,6 +455,84 @@ public class MamMigrationSelective implements CommandLineRunner { } } + + private void uploadFilesForEntry(Entry entry) { + + List downloadSelection = new ArrayList(); + + // Select download: Download original only. If original is not available, download largest file + Download selectedDownload = null; + for (Download download : entry.downloadList) { + if (download.entryFileVariation.equals("ORIGINAL")) { + selectedDownload = download; + break; + } + + if (selectedDownload == null) { + selectedDownload = download; + } + else if (selectedDownload.size < download.size) { + selectedDownload = download; + } + } + + if (selectedDownload != null) + downloadSelection.add(selectedDownload); + + for (Download download : entry.downloadListPublicFolder) { + downloadSelection.add(download); + } + + // Clean download selection + HashSet hrefs = new HashSet(); + List cleanedDownloads = new ArrayList(); + for (Download download : downloadSelection) { + if (hrefs.add(download.href)) { + cleanedDownloads.add(download); + } + } + + // Instead of downloading from MAM, take the pre-downloaded files from the harddrive + for (Download download : cleanedDownloads) { + String filenameToUse; + if (download.fileNameOnDisc != null) + filenameToUse = download.fileNameOnDisc; + else + filenameToUse = download.fileName; + + String filepath = "E:\\" + entry.id + "\\" + filenameToUse; + File uploadFile = new File(filepath); + + if (uploadFile.exists()) { + if (uploadFile.length() < 4000000000L) { + log.debug("filesize " + uploadFile.length() + " < 4 GB"); + + log.info("Uploading file " + filepath); + + MultiValueMap parameters = new LinkedMultiValueMap(); + parameters.add("file", new FileSystemResource(uploadFile)); + + String result = restTemplate.postForObject(adamHost + "/uploads", + new HttpEntity>(parameters, adamUploadHeaders), String.class); + +// String answer = restTemplate.exchange(adamHost + "/uploads", HttpMethod.POST, +// new HttpEntity>(parameters, adamUploadHeaders), String.class).getBody(); + + log.info("Upload result: " + result); + + } + else { + log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!"); + } + } + else { + log.error("Could not find file on disk " + filepath); + } + } + + + } + private void readEntryIDsFromFile() throws IOException { File entryIdListFile = new File("resources/entrylist_2017-10-25.txt"); log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath()); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java b/AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java index cdc8004..829dfe3 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/attic/MediaAssistantMigrationSimple.java @@ -22,10 +22,10 @@ import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; -import com.bayer.edam.migration.LoggingRequestInterceptor; import com.bayer.edam.migration.adam.ReadRecord; import com.bayer.edam.migration.adam.Token; import com.bayer.edam.migration.adam.UploadRecord; +import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor; import com.bayer.edam.migration.mam.Entry; public class MediaAssistantMigrationSimple { diff --git a/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java b/AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java similarity index 95% rename from AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java rename to AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java index 368b48a..2cb8eb5 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/interceptors/LoggingRequestInterceptor.java @@ -1,4 +1,4 @@ -package com.bayer.edam.migration; +package com.bayer.edam.migration.interceptors; import java.io.BufferedReader; import java.io.IOException;