From bb74e36db7f0a27cb15daca629a129df5d360751 Mon Sep 17 00:00:00 2001 From: GEXCE Date: Wed, 8 Nov 2017 15:42:27 +0100 Subject: [PATCH] 90% of file upload, GPC number --- .../edam/migration/MamMigrationSelective.java | 146 ++++++++++++------ .../bayer/edam/migration/UploadReference.java | 15 ++ .../bayer/edam/migration/attic/ZipTest.java | 31 ++++ 3 files changed, 147 insertions(+), 45 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 7156d43..243a8bd 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -1,13 +1,17 @@ package com.bayer.edam.migration; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -50,6 +54,9 @@ import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry.Download; import com.fasterxml.jackson.databind.ObjectMapper; +import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.model.FileHeader; + @SpringBootApplication @PropertySource("file:secret.properties") public class MamMigrationSelective implements CommandLineRunner { @@ -59,6 +66,8 @@ public class MamMigrationSelective implements CommandLineRunner { static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; private List entrySelection = new ArrayList(Arrays.asList( 71263 /*72048, 60984 /*48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984 + static final String fileCache = "E:\\"; + static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; @@ -80,21 +89,17 @@ public class MamMigrationSelective implements CommandLineRunner { private List campaigns = new LinkedList(); private CategoryTree mamCategoryTree; - - static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss"); - static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log"); static final String fieldSeparator = "|~|"; static final String lineSeparator = "\r\n"; + private List openUploads = new ArrayList(); + @Override public void run(String... strings) throws Exception { // XXX // readEntryIDsFromFile(); - log.info("Write log headers for " + entryLog.getAbsolutePath()); - FileUtils.writeStringToFile(entryLog, "entryId" + fieldSeparator + "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8")); - log.info("Read categories"); mamCategoryTree = new CategoryTree(categoryJson, categoryMapping); @@ -407,7 +412,15 @@ public class MamMigrationSelective implements CommandLineRunner { // Keywords if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) { - keywords.addAll(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"))); + List mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")); + + for (String s : mamKeywords) { + keywords.add(s); + + // if the keyword is four digits - dash - four digits, it's a GPC number + if (s.matches("\\b\\d{4}-\\d{4}\\b")) + record.addField("byr_ah_approvalNumber", s); + } } if (keywords.size() > 0) { @@ -440,16 +453,15 @@ public class MamMigrationSelective implements CommandLineRunner { HttpEntity httpEntity = new HttpEntity(record, adamHeaders); try { -// URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); -// log.info("RESULT Entry " + entry.id + " migrated to record " + location); + URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); + log.info("RESULT Entry " + entry.id + " migrated to record " + location); // XXX - uploadFilesForEntry(entry); + uploadFilesForEntry(entry, location); // TODO process file upload errors -// FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator, -// Charset.forName("utf-8"), true); + log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); } catch (HttpStatusCodeException hsce) { log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString()); @@ -458,9 +470,9 @@ public class MamMigrationSelective implements CommandLineRunner { } - private void uploadFilesForEntry(Entry entry) throws Exception { - - List downloadSelection = new ArrayList(); + private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception { + HashSet hrefsForUpload = new HashSet(); + List filesForUpload = new ArrayList(); // Select download: Download original only. If original is not available, download largest file Download selectedDownload = null; @@ -478,62 +490,106 @@ public class MamMigrationSelective implements CommandLineRunner { } } - if (selectedDownload != null) - downloadSelection.add(selectedDownload); - - for (Download download : entry.downloadListPublicFolder) { - downloadSelection.add(download); + if (selectedDownload != null) { + + // if selected download is a zip that contains a single file, upload extracted file as master + if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) { + File masterFile = getFileForEntry(selectedDownload, entry.id); + + ZipFile zipFile = new ZipFile(masterFile); + @SuppressWarnings("unchecked") + List fileHeaders = zipFile.getFileHeaders(); + + if (fileHeaders.size() == 1) { + String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName(); + + if (!(new File(extractedMasterFilePath).exists())) { + log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName()); + zipFile.extractAll(fileCache + entry.id + "\\"); + } + log.info("Using extracted master file " + extractedMasterFilePath); + masterFile = new File(extractedMasterFilePath); + } + + filesForUpload.add(masterFile); + hrefsForUpload.add(selectedDownload.href); + } + } + else { + log.warn("Could not identify master file for entry " + entry.id); } - // Clean download selection - HashSet hrefs = new HashSet(); - List cleanedDownloads = new ArrayList(); - for (Download download : downloadSelection) { - if (hrefs.add(download.href)) { - cleanedDownloads.add(download); + for (Download download : entry.downloadListPublicFolder) { + if (hrefsForUpload.add(download.href)) { + filesForUpload.add(getFileForEntry(download, entry.id)); } } // Instead of downloading from MAM, take the pre-downloaded files from the harddrive - for (Download download : cleanedDownloads) { - String filenameToUse; - if (download.fileNameOnDisc != null) - filenameToUse = download.fileNameOnDisc; - else - filenameToUse = download.fileName; - - String filepath = "E:\\" + entry.id + "\\" + filenameToUse; - File uploadFile = new File(filepath); - - if (uploadFile.exists()) { - if (uploadFile.length() < 4000000000L) { - log.info("Uploading file " + filepath + ", size " + uploadFile.length()); + for (File file : filesForUpload) { + if (file.exists()) { + if (file.length() < 4000000000L) { + log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length()); MultiValueMap parameters = new LinkedMultiValueMap(); - parameters.add("file", new FileSystemResource(uploadFile)); + parameters.add("file", new FileSystemResource(file)); UploadResult result = restTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST, new HttpEntity>(parameters, adamUploadHeaders), UploadResult.class).getBody(); if (result.token != null) { + // TODO + log.debug("Token: " + result.token); } - - log.info("Upload result: " + result); + else if (result.uri != null) { + log.debug("Storing upload reference URI " + result.uri); + openUploads.add(new UploadReference(recordLocation, result.uri)); + } + else { + log.error("No token/URI after upload!"); + } } else { - log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!"); + // TODO + + log.debug("filesize " + file.length() + " > 4 GB!!!1!"); } } else { - log.error("Could not find file on disk " + filepath); + log.error("Could not find file on disk " + file.getAbsolutePath()); } } } + private File getFileForEntry(Download download, Long id) throws IOException { + String filenameToUse; + if (download.fileNameOnDisc != null) + filenameToUse = download.fileNameOnDisc; + else + filenameToUse = download.fileName; + + File file = new File(fileCache + id + "\\" + filenameToUse); + + // check whether file is in cache, download if it's not there + if (!file.exists()) { + log.debug("Downloading missing file " + filenameToUse + " for entry " + id); + URL fileLink = new URL(download.href); + ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream()); + FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + } + else { + log.debug("Using existing file " + filenameToUse + " for entry " + id); + } + + return file; + } + private void readEntryIDsFromFile() throws IOException { File entryIdListFile = new File("resources/entrylist_2017-10-25.txt"); log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath()); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java b/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java new file mode 100644 index 0000000..6a5c830 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java @@ -0,0 +1,15 @@ +package com.bayer.edam.migration; + +import java.net.URI; + +public class UploadReference { + + URI targetRecord; + String fileURI; + + public UploadReference(URI record, String fileURI) { + this.targetRecord = record; + this.fileURI = fileURI; + } + +} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java b/AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java new file mode 100644 index 0000000..5f57790 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java @@ -0,0 +1,31 @@ +package com.bayer.edam.migration.attic; + +import java.util.List; + +import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.model.FileHeader; + +public class ZipTest { + + public static void main(String[] args) { + try { + ZipFile zipFile = new ZipFile("C:\\Temp\\Migration AH\\ziptest\\folder.zip"); + List fileList = zipFile.getFileHeaders(); + + System.out.println(fileList.size()); + + for (FileHeader header : fileList) { + System.out.println(header.getFileName() + ", " + header.getUncompressedSize()); + } + + System.out.println(fileList.get(0).getFileName()); + + System.out.println("test".matches("\\d\\d\\d\\d-\\d\\d\\d\\d\\b")); + System.out.println("2121-5531".matches("\\b\\d{4}-\\d{4}\\b")); + + } catch (Exception e) { + e.printStackTrace(); + } + } + +}