diff --git a/AH MAM Migration/questions.txt b/AH MAM Migration/questions.txt index 72c6a7a..c3ac221 100644 --- a/AH MAM Migration/questions.txt +++ b/AH MAM Migration/questions.txt @@ -41,12 +41,17 @@ - Englisch OK - Additional Info in einzelnen Zeilen [done] - Comment und Description [done] -- Status: Liste mit Status != ACCEPTED an Claudia/Heike +- Status: Liste mit Status != ACCEPTED an Claudia/Heike [done] Archiv ist nur andere Kategorie -- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess +- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin/Tilman klären Prozess - Seite 2 als Preview - Filename Preview nach Feld fileName setzen ----------------------------------------------------- +Status „Archived“ kann gelöscht werden bzw. muss nicht übernommen werden. +Heike prüft nochmal den Status „angelegt“ und gibt dann Rückmeldung. +Status „gesperrt“ kann gelöscht werden bzw. muss nicht übernommen werden. +Status „Zur Indexierung zugewiesen“ kann gelöscht bzw. muss nicht übernommen werden. +Status „Zum Upload zugewiesen“ bitte behalten. diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index c6ec16e..b1ed2f0 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -1,4 +1,4 @@ # TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF logging.level.com.bayer=DEBUG logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=migration.log \ No newline at end of file +logging.file=entryDownload.log \ No newline at end of file diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java index ca7bb8e..9f90254 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java @@ -1,5 +1,13 @@ package com.bayer.edam.migration; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -17,7 +25,7 @@ import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entrylist; import com.fasterxml.jackson.databind.ObjectMapper; -//@SpringBootApplication +@SpringBootApplication @PropertySource("file:secret.properties") public class EntryDownloader implements CommandLineRunner { @@ -37,7 +45,7 @@ public class EntryDownloader implements CommandLineRunner { @Override - public void run(String... strings) throws Exception { + public void run(String... strings) { HttpHeaders headers = new HttpHeaders(); headers.add("Accept", "application/json"); headers.add("Authorization", "Basic " + mamAuth); @@ -51,18 +59,69 @@ public class EntryDownloader implements CommandLineRunner { lastPage = entrylist.lastPage; for (int i = 0; i < entrylist.entries.length; i++) { - String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody(); - Entry entry = jsonMapper.readValue(entryJson, Entry.class); - - long aggregatedFilesize = 0; - for (Download download : entry.downloadList) { - aggregatedFilesize += download.size; + try { + String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody(); + Entry entry = jsonMapper.readValue(entryJson, Entry.class); + + List downloadSelection = new ArrayList(); + + // Select download: Download original only. If original is not available, download largest file + Download selectedDownload = null; + for (Download download : entry.downloadList) { + if (download.entryFileVariation.equals("ORIGINAL")) { + selectedDownload = download; + break; + } + + if (selectedDownload == null) { + selectedDownload = download; + } + else if (selectedDownload.size < download.size) { + selectedDownload = download; + } + } + + if (selectedDownload != null) + downloadSelection.add(selectedDownload); + + // TODO extract, if zip --> postponed + + for (Download download : entry.downloadListPublicFolder) { + downloadSelection.add(download); + } + + // clean download selection + HashSet hrefs = new HashSet(); + List cleanedDownloads = new ArrayList(); + for (Download download : downloadSelection) { + if (hrefs.add(download.href)) { + cleanedDownloads.add(download); + } + } + + long aggregatedFilesize = 0; + for (Download download : cleanedDownloads) { + aggregatedFilesize += download.size; + } + + log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson); + + new File("E:\\" + entry.id).mkdir(); + + for (Download download : cleanedDownloads) { + String filenameToUse; + if (download.fileNameOnDisc != null) + filenameToUse = download.fileNameOnDisc; + else + filenameToUse = download.fileName; + + FileUtils.copyURLToFile(new URL(download.href), new File("E:\\" + entry.id + "\\" + filenameToUse), 4000, 240000); + } } - for (Download download : entry.downloadListPublicFolder) { - aggregatedFilesize += download.size; + catch (IOException ioe) { + log.error(ioe.toString()); + ioe.printStackTrace(); } - - log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson); } } } diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java index 1de0328..4405fd7 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java @@ -2,7 +2,9 @@ package com.bayer.edam.migration; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.commons.io.LineIterator; @@ -10,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.bayer.edam.migration.mam.Entry; +import com.bayer.edam.migration.mam.Entry.Download; import com.fasterxml.jackson.databind.ObjectMapper; public class EntryProcessor { @@ -30,62 +33,98 @@ public class EntryProcessor { // skip header iter.next(); - HashSet orgSet = new HashSet(); int count = 0; + long aggregatedFilesize = 0; while (iter.hasNext()) { String line = iter.next(); line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length()); Entry entry = jsonMapper.readValue(line, Entry.class); - if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) { - - String orgId = null; - - if (entry.copyright != null && entry.copyright.organization != null) { - orgId = entry.copyright.organization.get("id"); - orgSet.add(orgId); + List downloadSelection = new ArrayList(); + Download selectedDownload = null; + for (Download download : entry.downloadList) { + if (download.entryFileVariation.equals("ORIGINAL")) { + selectedDownload = download; + break; } - - log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId + ", status: " + entry.stateId); + if (selectedDownload == null) { + selectedDownload = download; + } + else if (selectedDownload.size < download.size) { + selectedDownload = download; + } } + + if (selectedDownload != null) + downloadSelection.add(selectedDownload); + + for (Download download : entry.downloadListPublicFolder) { + downloadSelection.add(download); + } + + HashSet hrefs = new HashSet(); + List cleanedDownloads = new ArrayList(); + for (Download download : downloadSelection) { + if (hrefs.add(download.href)) { + cleanedDownloads.add(download); + } + } + + HashSet filenames = new HashSet(); + for (Download download : cleanedDownloads) { + String filenameToUse; + if (download.fileNameOnDisc != null) + filenameToUse = download.fileNameOnDisc; + else + filenameToUse = download.fileName; + + if (!filenames.add(filenameToUse)) { + log.error(entry.id + ": " + filenameToUse); + System.exit(1); + } + + aggregatedFilesize += download.size; + } + + + /* + HashSet filenames = new HashSet(); + HashSet hrefs = new HashSet(); + for (Download download : downloadSelection) { + String filenameToUse; + if (download.fileNameOnDisc != null) + filenameToUse = download.fileNameOnDisc; + else + filenameToUse = download.fileName; + + if (filenameToUse == null) { + log.error("ARGH " + entry.id); + System.exit(1); + } + + if (filenames.add(filenameToUse)) { + aggregatedFilesize += download.size; + } + hrefs.add(download.href); + } -// for (Download download : entry.downloadListPublicFolder) { -// -// } - -// boolean originalFound = false; -// long largest = 0; -// long originalSize = 0; -// for (Download download : entry.downloadList) { -// if (largest < download.size) -// largest = download.size; -// -// if (download.entryFileVariation.equals("ORIGINAL")) { -// originalFound = true; -// originalSize = download.size; -// } -// } -// -// if (!originalFound) { -// log.info(entry.id + " - no original - " + entry.stateId); -// count++; -// } -// else if (largest > originalSize) { -// log.info(entry.id + " - original is not largest file - " + entry.stateId); -// count++; -// } + if (filenames.size() < hrefs.size()) { + log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size()); + for (String s : filenames) + log.error(s); + for (String s : hrefs) + log.error(s); + } + */ } - for (String orgId : orgSet) { - log.info(orgId); - } - log.info("Count: " + count); + log.info("aggregatedFilesize: " + aggregatedFilesize); } diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 47db4da..6c97ceb 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -88,7 +88,7 @@ public class MamMigrationSelective implements CommandLineRunner { Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); - postToAdam(entry); +// postToAdam(entry); } // // delete all created records