EntryDownloader with files

AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java

@@ -2,7 +2,9 @@ package com.bayer.edam.migration;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.LineIterator;
@@ -10,6 +12,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.bayer.edam.migration.mam.Entry;
+import com.bayer.edam.migration.mam.Entry.Download;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 public class EntryProcessor {
@@ -30,62 +33,98 @@ public class EntryProcessor {
 		// skip header
 		iter.next();
 		
-		HashSet<String> orgSet = new HashSet<String>();
 		int count = 0;
+		long aggregatedFilesize = 0;
 		
 		while (iter.hasNext()) {
 			String line = iter.next();
 			line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
 			Entry entry = jsonMapper.readValue(line, Entry.class);
 			
-			if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
-				
-				String orgId = null;
-				
-				if (entry.copyright != null && entry.copyright.organization != null) {
-					orgId = entry.copyright.organization.get("id");
-					orgSet.add(orgId);
+			List<Download> downloadSelection = new ArrayList<Download>();
+			Download selectedDownload = null;
+			for (Download download : entry.downloadList) {
+				if (download.entryFileVariation.equals("ORIGINAL")) {
+					selectedDownload = download;
+					break;
 				}
 				
-				
-				log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId + ", status: " + entry.stateId);
+				if (selectedDownload == null) {
+					selectedDownload = download;
+				}
+				else if (selectedDownload.size < download.size) {
+					selectedDownload = download;
+				}
 			}
+			
+			if (selectedDownload != null)
+				downloadSelection.add(selectedDownload);
+			
+			for (Download download : entry.downloadListPublicFolder) {
+				downloadSelection.add(download);
+			}
+			
+			HashSet<String> hrefs = new HashSet<String>();
+			List<Download> cleanedDownloads = new ArrayList<Download>();
+			for (Download download : downloadSelection) {
+				if (hrefs.add(download.href)) {
+					cleanedDownloads.add(download);
+				}
+			}
+			
+			HashSet<String> filenames = new HashSet<String>();
+			for (Download download : cleanedDownloads) {
+				String filenameToUse;
+				if (download.fileNameOnDisc != null)
+					filenameToUse = download.fileNameOnDisc;
+				else
+					filenameToUse = download.fileName;
+				
+				if (!filenames.add(filenameToUse)) {
+					log.error(entry.id + ": " + filenameToUse);
+					System.exit(1);
+				}
+				
+				aggregatedFilesize += download.size;
+			}
+			
+			
 
 			
+			/*
+			HashSet<String> filenames = new HashSet<String>();
+			HashSet<String> hrefs = new HashSet<String>();
+			for (Download download : downloadSelection) {
+				String filenameToUse;
+				if (download.fileNameOnDisc != null)
+					filenameToUse = download.fileNameOnDisc;
+				else
+					filenameToUse = download.fileName;
+				
+				if (filenameToUse == null) {
+					log.error("ARGH " + entry.id);
+					System.exit(1);
+				}
+				
+				if (filenames.add(filenameToUse)) {
+					aggregatedFilesize += download.size;
+				}
+				hrefs.add(download.href);
+			}
 			
-//			for (Download download : entry.downloadListPublicFolder) {
-//				
-//			}
-			
-//			boolean originalFound = false;
-//			long largest = 0;
-//			long originalSize = 0;
-//			for (Download download : entry.downloadList) {
-//				if (largest < download.size)
-//					largest = download.size;
-//				
-//				if (download.entryFileVariation.equals("ORIGINAL")) {
-//					originalFound = true;
-//					originalSize = download.size;
-//				}
-//			}
-//			
-//			if (!originalFound) {
-//				log.info(entry.id + " - no original - " + entry.stateId);
-//				count++;
-//			}
-//			else if (largest > originalSize) {
-//				log.info(entry.id + " - original is not largest file - " + entry.stateId);
-//				count++;
-//			}
+			if (filenames.size() < hrefs.size()) {
+				log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size());
+				for (String s : filenames)
+					log.error(s);
+				for (String s : hrefs)
+					log.error(s);
+			}
+			*/
 
 		}
 		
-		for (String orgId : orgSet) {
-			log.info(orgId);
-		}
-		
 		log.info("Count: " + count);
+		log.info("aggregatedFilesize: " + aggregatedFilesize);
 
 	}