EntryDownloader with files

AH MAM Migration/questions.txt

@@ -41,12 +41,17 @@
 - Englisch OK
 - Additional Info in einzelnen Zeilen [done]
 - Comment und Description [done]
-- Status: Liste mit Status != ACCEPTED an Claudia/Heike
+- Status: Liste mit Status != ACCEPTED an Claudia/Heike [done]
   Archiv ist nur andere Kategorie
-- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess
+- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin/Tilman klären Prozess
 - Seite 2 als Preview
 - Filename Preview nach Feld fileName setzen
 -----------------------------------------------------
+Status „Archived“ kann gelöscht werden bzw. muss nicht übernommen werden.
+Heike prüft nochmal den Status „angelegt“ und gibt dann Rückmeldung.
+Status „gesperrt“ kann gelöscht werden bzw. muss nicht übernommen werden.
+Status „Zur Indexierung zugewiesen“ kann gelöscht bzw. muss nicht übernommen werden.
+Status „Zum Upload zugewiesen“ bitte behalten.
 
 
 

AH MAM Migration/resources/application.properties

@@ -1,4 +1,4 @@
 # TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
 logging.level.com.bayer=DEBUG
 logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
-logging.file=migration.log
+logging.file=entryDownload.log

AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java

@@ -1,5 +1,13 @@
 package com.bayer.edam.migration;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -17,7 +25,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
 import com.bayer.edam.migration.mam.Entrylist;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-//@SpringBootApplication
+@SpringBootApplication
 @PropertySource("file:secret.properties")
 public class EntryDownloader implements CommandLineRunner {
 	
@@ -37,7 +45,7 @@ public class EntryDownloader implements CommandLineRunner {
 	
 	
 	@Override
-	public void run(String... strings) throws Exception {
+	public void run(String... strings) {
 		HttpHeaders headers = new HttpHeaders();
 		headers.add("Accept", "application/json");
 		headers.add("Authorization", "Basic " + mamAuth);
@@ -51,18 +59,69 @@ public class EntryDownloader implements CommandLineRunner {
 			lastPage = entrylist.lastPage;
 			
 			for (int i = 0; i < entrylist.entries.length; i++) {
-				String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
+				try {
-				Entry entry = jsonMapper.readValue(entryJson, Entry.class);
+					String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
-				
+					Entry entry = jsonMapper.readValue(entryJson, Entry.class);
-				long aggregatedFilesize = 0;
+					
-				for (Download download : entry.downloadList) {
+					List<Download> downloadSelection = new ArrayList<Download>();
-					aggregatedFilesize += download.size;
+					
+					// Select download: Download original only. If original is not available, download largest file
+					Download selectedDownload = null;
+					for (Download download : entry.downloadList) {
+						if (download.entryFileVariation.equals("ORIGINAL")) {
+							selectedDownload = download;
+							break;
+						}
+						
+						if (selectedDownload == null) {
+							selectedDownload = download;
+						}
+						else if (selectedDownload.size < download.size) {
+							selectedDownload = download;
+						}
+					}
+					
+					if (selectedDownload != null)
+						downloadSelection.add(selectedDownload);
+					
+					// TODO extract, if zip --> postponed
+					
+					for (Download download : entry.downloadListPublicFolder) {
+						downloadSelection.add(download);
+					}
+					
+					// clean download selection
+					HashSet<String> hrefs = new HashSet<String>();
+					List<Download> cleanedDownloads = new ArrayList<Download>();
+					for (Download download : downloadSelection) {
+						if (hrefs.add(download.href)) {
+							cleanedDownloads.add(download);
+						}
+					}
+					
+					long aggregatedFilesize = 0;
+					for (Download download : cleanedDownloads) {
+						aggregatedFilesize += download.size;
+					}
+					
+					log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
+					
+					new File("E:\\" + entry.id).mkdir();
+					
+					for (Download download : cleanedDownloads) {
+						String filenameToUse;
+						if (download.fileNameOnDisc != null)
+							filenameToUse = download.fileNameOnDisc;
+						else
+							filenameToUse = download.fileName;
+
+						FileUtils.copyURLToFile(new URL(download.href), new File("E:\\" + entry.id + "\\" + filenameToUse), 4000, 240000);
+					}
 				}
-				for (Download download : entry.downloadListPublicFolder) {
+				catch (IOException ioe) {
-					aggregatedFilesize += download.size;
+					log.error(ioe.toString());
+					ioe.printStackTrace();
 				}
-				
-				log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
 			}
 		}
 	}

AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java

@@ -2,7 +2,9 @@ package com.bayer.edam.migration;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.LineIterator;
@@ -10,6 +12,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.bayer.edam.migration.mam.Entry;
+import com.bayer.edam.migration.mam.Entry.Download;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 public class EntryProcessor {
@@ -30,62 +33,98 @@ public class EntryProcessor {
 		// skip header
 		iter.next();
 		
-		HashSet<String> orgSet = new HashSet<String>();
 		int count = 0;
+		long aggregatedFilesize = 0;
 		
 		while (iter.hasNext()) {
 			String line = iter.next();
 			line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
 			Entry entry = jsonMapper.readValue(line, Entry.class);
 			
-			if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
+			List<Download> downloadSelection = new ArrayList<Download>();
-				
+			Download selectedDownload = null;
-				String orgId = null;
+			for (Download download : entry.downloadList) {
-				
+				if (download.entryFileVariation.equals("ORIGINAL")) {
-				if (entry.copyright != null && entry.copyright.organization != null) {
+					selectedDownload = download;
-					orgId = entry.copyright.organization.get("id");
+					break;
-					orgSet.add(orgId);
 				}
 				
-				
+				if (selectedDownload == null) {
-				log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId + ", status: " + entry.stateId);
+					selectedDownload = download;
+				}
+				else if (selectedDownload.size < download.size) {
+					selectedDownload = download;
+				}
 			}
+			
+			if (selectedDownload != null)
+				downloadSelection.add(selectedDownload);
+			
+			for (Download download : entry.downloadListPublicFolder) {
+				downloadSelection.add(download);
+			}
+			
+			HashSet<String> hrefs = new HashSet<String>();
+			List<Download> cleanedDownloads = new ArrayList<Download>();
+			for (Download download : downloadSelection) {
+				if (hrefs.add(download.href)) {
+					cleanedDownloads.add(download);
+				}
+			}
+			
+			HashSet<String> filenames = new HashSet<String>();
+			for (Download download : cleanedDownloads) {
+				String filenameToUse;
+				if (download.fileNameOnDisc != null)
+					filenameToUse = download.fileNameOnDisc;
+				else
+					filenameToUse = download.fileName;
+				
+				if (!filenames.add(filenameToUse)) {
+					log.error(entry.id + ": " + filenameToUse);
+					System.exit(1);
+				}
+				
+				aggregatedFilesize += download.size;
+			}
+			
+			
 
 			
+			/*
+			HashSet<String> filenames = new HashSet<String>();
+			HashSet<String> hrefs = new HashSet<String>();
+			for (Download download : downloadSelection) {
+				String filenameToUse;
+				if (download.fileNameOnDisc != null)
+					filenameToUse = download.fileNameOnDisc;
+				else
+					filenameToUse = download.fileName;
+				
+				if (filenameToUse == null) {
+					log.error("ARGH " + entry.id);
+					System.exit(1);
+				}
+				
+				if (filenames.add(filenameToUse)) {
+					aggregatedFilesize += download.size;
+				}
+				hrefs.add(download.href);
+			}
 			
-//			for (Download download : entry.downloadListPublicFolder) {
+			if (filenames.size() < hrefs.size()) {
-//				
+				log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size());
-//			}
+				for (String s : filenames)
-			
+					log.error(s);
-//			boolean originalFound = false;
+				for (String s : hrefs)
-//			long largest = 0;
+					log.error(s);
-//			long originalSize = 0;
+			}
-//			for (Download download : entry.downloadList) {
+			*/
-//				if (largest < download.size)
-//					largest = download.size;
-//				
-//				if (download.entryFileVariation.equals("ORIGINAL")) {
-//					originalFound = true;
-//					originalSize = download.size;
-//				}
-//			}
-//			
-//			if (!originalFound) {
-//				log.info(entry.id + " - no original - " + entry.stateId);
-//				count++;
-//			}
-//			else if (largest > originalSize) {
-//				log.info(entry.id + " - original is not largest file - " + entry.stateId);
-//				count++;
-//			}
 
 		}
 		
-		for (String orgId : orgSet) {
-			log.info(orgId);
-		}
-		
 		log.info("Count: " + count);
+		log.info("aggregatedFilesize: " + aggregatedFilesize);
 
 	}
 

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -88,7 +88,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 
 			Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
 
-			postToAdam(entry);
+//			postToAdam(entry);
 		}
 		
 //		// delete all created records