90% of file upload, GPC number

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -1,13 +1,17 @@
 package com.bayer.edam.migration;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.charset.Charset;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -50,6 +54,9 @@ import com.bayer.edam.migration.mam.Entry;
 import com.bayer.edam.migration.mam.Entry.Download;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import net.lingala.zip4j.core.ZipFile;
+import net.lingala.zip4j.model.FileHeader;
+
 @SpringBootApplication
 @PropertySource("file:secret.properties")
 public class MamMigrationSelective implements CommandLineRunner {
@@ -59,6 +66,8 @@ public class MamMigrationSelective implements CommandLineRunner {
 	static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
 	private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 71263 /*72048, 60984 /*48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
 	
+	static final String fileCache = "E:\\";
+	
 	static final String categoryJson = "resources/2017-10-19_categories.json";
 	static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
 
@@ -80,21 +89,17 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private List<Campaign> campaigns = new LinkedList<Campaign>();
 	private CategoryTree mamCategoryTree;
 	
-	
-	static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
-	static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log");
 	static final String fieldSeparator = "|~|";
 	static final String lineSeparator = "\r\n";
 	
+	private List<UploadReference> openUploads = new ArrayList<UploadReference>();
+	
 	@Override
 	public void run(String... strings) throws Exception {
 		
 		// XXX
 //		readEntryIDsFromFile();
 		
-		log.info("Write log headers for " + entryLog.getAbsolutePath());
-		FileUtils.writeStringToFile(entryLog, "entryId" + fieldSeparator + "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
-		
 		log.info("Read categories");
 		mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
 		
@@ -407,7 +412,15 @@ public class MamMigrationSelective implements CommandLineRunner {
 
 		// Keywords
 		if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
-			keywords.addAll(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
+			List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
+			
+			for (String s : mamKeywords) {
+				keywords.add(s);
+				
+				// if the keyword is four digits - dash - four digits, it's a GPC number
+				if (s.matches("\\b\\d{4}-\\d{4}\\b"))
+					record.addField("byr_ah_approvalNumber", s);
+			}
 		}
 		
 		if (keywords.size() > 0) {
@@ -440,16 +453,15 @@ public class MamMigrationSelective implements CommandLineRunner {
 		HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
 		try {
 			
-//			URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
+			URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
-//			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
+			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
 			
 			// XXX
-			uploadFilesForEntry(entry);
+			uploadFilesForEntry(entry, location);
 			
 			// TODO process file upload errors
 			
-//			FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
+			log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
-//					Charset.forName("utf-8"), true);
 		}
 		catch (HttpStatusCodeException hsce) {
 			log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -458,9 +470,9 @@ public class MamMigrationSelective implements CommandLineRunner {
 	}
 	
 	
-	private void uploadFilesForEntry(Entry entry) throws Exception {
+	private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
-		
+		HashSet<String> hrefsForUpload = new HashSet<String>();
-		List<Download> downloadSelection = new ArrayList<Download>();
+		List<File> filesForUpload = new ArrayList<File>();
 		
 		// Select download: Download original only. If original is not available, download largest file
 		Download selectedDownload = null;
@@ -478,62 +490,106 @@ public class MamMigrationSelective implements CommandLineRunner {
 			}
 		}
 		
-		if (selectedDownload != null)
+		if (selectedDownload != null) {
-			downloadSelection.add(selectedDownload);
+			
-		
+			// if selected download is a zip that contains a single file, upload extracted file as master
-		for (Download download : entry.downloadListPublicFolder) {
+			if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) {
-			downloadSelection.add(download);
+				File masterFile = getFileForEntry(selectedDownload, entry.id);
+				
+				ZipFile zipFile = new ZipFile(masterFile);
+				@SuppressWarnings("unchecked")
+				List<FileHeader> fileHeaders = zipFile.getFileHeaders();
+				
+				if (fileHeaders.size() == 1) {
+					String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
+					
+					if (!(new File(extractedMasterFilePath).exists())) {
+						log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName());
+						zipFile.extractAll(fileCache + entry.id + "\\");
+					}
+					log.info("Using extracted master file " + extractedMasterFilePath);
+					masterFile = new File(extractedMasterFilePath);
+				}
+				
+				filesForUpload.add(masterFile);
+				hrefsForUpload.add(selectedDownload.href);
+			}
+		}
+		else {
+			log.warn("Could not identify master file for entry " + entry.id);
 		}
 		
-		// Clean download selection
+		for (Download download : entry.downloadListPublicFolder) {
-		HashSet<String> hrefs = new HashSet<String>();
+			if (hrefsForUpload.add(download.href)) {
-		List<Download> cleanedDownloads = new ArrayList<Download>();
+				filesForUpload.add(getFileForEntry(download, entry.id));
-		for (Download download : downloadSelection) {
-			if (hrefs.add(download.href)) {
-				cleanedDownloads.add(download);
 			}
 		}
 		
 		// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
-		for (Download download : cleanedDownloads) {
+		for (File file : filesForUpload) {
-			String filenameToUse;
+			if (file.exists()) {
-			if (download.fileNameOnDisc != null)
+				if (file.length() < 4000000000L) {
-				filenameToUse = download.fileNameOnDisc;
+					log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
-			else
-				filenameToUse = download.fileName;
-			
-			String filepath = "E:\\" + entry.id + "\\" + filenameToUse;
-			File uploadFile = new File(filepath);
-			
-			if (uploadFile.exists()) {
-				if (uploadFile.length() < 4000000000L) {
-					log.info("Uploading file " + filepath + ", size " + uploadFile.length());
 					
 					MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
-					parameters.add("file", new FileSystemResource(uploadFile));
+					parameters.add("file", new FileSystemResource(file));
 
 					UploadResult result = restTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
 					new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
 					
 					if (result.token != null) {
+						// TODO
+						
 						log.debug("Token: " + result.token);
 					}
-					
+					else if (result.uri != null) {
-					log.info("Upload result: " + result);
+						log.debug("Storing upload reference URI " + result.uri);
+						openUploads.add(new UploadReference(recordLocation, result.uri));
+					}
+					else {
+						log.error("No token/URI after upload!");
+					}
 					
 				}
 				else {
-					log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!");
+					// TODO
+					
+					log.debug("filesize " + file.length() + " > 4 GB!!!1!");
 				}
 			}
 			else {
-				log.error("Could not find file on disk " + filepath);
+				log.error("Could not find file on disk " + file.getAbsolutePath());
 			}
 		}
 
 		
 	}
 
+	private File getFileForEntry(Download download, Long id) throws IOException {
+		String filenameToUse;
+		if (download.fileNameOnDisc != null)
+			filenameToUse = download.fileNameOnDisc;
+		else
+			filenameToUse = download.fileName;
+		
+		File file = new File(fileCache + id + "\\" + filenameToUse);
+		
+		// check whether file is in cache, download if it's not there
+		if (!file.exists()) {
+			log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
+			URL fileLink = new URL(download.href);
+			ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
+			FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
+			fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+			fos.close();
+		}
+		else {
+			log.debug("Using existing file " + filenameToUse + " for entry " + id);
+		}
+		
+		return file;
+	}
+
 	private void readEntryIDsFromFile() throws IOException {
 		File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
 		log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());

AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java

@@ -0,0 +1,15 @@
+package com.bayer.edam.migration;
+
+import java.net.URI;
+
+public class UploadReference {
+	
+	URI targetRecord;
+	String fileURI;
+	
+	public UploadReference(URI record, String fileURI) {
+		this.targetRecord = record;
+		this.fileURI = fileURI;
+	}
+
+}

AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java

@@ -0,0 +1,31 @@
+package com.bayer.edam.migration.attic;
+
+import java.util.List;
+
+import net.lingala.zip4j.core.ZipFile;
+import net.lingala.zip4j.model.FileHeader;
+
+public class ZipTest {
+
+	public static void main(String[] args) {
+		try {
+			ZipFile zipFile = new ZipFile("C:\\Temp\\Migration AH\\ziptest\\folder.zip");
+			List<FileHeader> fileList = zipFile.getFileHeaders();
+			
+			System.out.println(fileList.size());
+			
+			for (FileHeader header : fileList) {
+				System.out.println(header.getFileName() + ", " + header.getUncompressedSize());
+			}
+			
+			System.out.println(fileList.get(0).getFileName());
+			
+			System.out.println("test".matches("\\d\\d\\d\\d-\\d\\d\\d\\d\\b"));
+			System.out.println("2121-5531".matches("\\b\\d{4}-\\d{4}\\b"));
+			
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+}