From d5866ac48cfc00d511361aafcf40b13321471d6c Mon Sep 17 00:00:00 2001 From: gexce Date: Fri, 17 Nov 2017 11:28:46 +0100 Subject: [PATCH] Fixed MAM download --- .../resources/application.properties | 2 +- .../com/bayer/edam/migration/Definitions.java | 2 +- .../bayer/edam/migration/EntryDownloader.java | 37 ++++++++++--------- .../edam/migration/MamMigrationSelective.java | 26 ++++++++----- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index 9f89b58..e3c2eee 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -2,4 +2,4 @@ logging.level.com.bayer=DEBUG #logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application_2017-11-16_QA_3_3.log +logging.file=application_2017-11-17.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java index e221b36..d8c27f8 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java @@ -49,7 +49,7 @@ public class Definitions { users = new HashMap(); // XXX QA values - classifications.put("Migration.Test", new WriteClassification("de8edf2ed8d2405b87dea82d01062d50", 1)); + classifications.put("Migration.Test", new WriteClassification("f7f19a55835e4936aa61a82d0116ae47", 1)); classifications.put("Migration", new WriteClassification("2bd24882-b7d8-4d30-8e6f-a7ef00eb33e7", 1)); classifications.put("Migration.Logos", new WriteClassification("de96f8db1de44cc6841ba81c00b9db8e", 1)); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java index fe98867..1e59695 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java @@ -27,12 +27,15 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.client.BufferingClientHttpRequestFactory; +import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; +import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor; import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entrylist; +import com.bayer.edam.migration.mam.Entrylist.EntryOverview; import com.fasterxml.jackson.databind.ObjectMapper; //@SpringBootApplication @@ -44,11 +47,11 @@ public class EntryDownloader implements CommandLineRunner { /** * If true, files are downloaded. If false, only the IDs are written into the outputfile. */ - private final boolean downloadFiles = false; + private final boolean downloadFiles = true; private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\"; - File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\entrylist_FULL_2017-11-15.txt"); + File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\DOWNLOAD_LIST.txt"); @Value("${mam.auth}") @@ -87,9 +90,9 @@ public class EntryDownloader implements CommandLineRunner { public void run(String... strings) { restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory())); -// List interceptors = new ArrayList(); -// interceptors.add(new LoggingRequestInterceptor()); -// restTemplate.setInterceptors(interceptors); + List interceptors = new ArrayList(); + interceptors.add(new LoggingRequestInterceptor()); + restTemplate.setInterceptors(interceptors); HttpHeaders headers = new HttpHeaders(); headers.add("Accept", "application/json"); @@ -103,25 +106,25 @@ public class EntryDownloader implements CommandLineRunner { int lastPage = 1; for (int pageNum = 1; pageNum <= lastPage; pageNum++) { - Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody(); - lastPage = entrylist.lastPage; - log.debug("processing page " + pageNum + " of " + lastPage); +// Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody(); +// lastPage = entrylist.lastPage; +// log.debug("processing page " + pageNum + " of " + lastPage); -// // XXX -// Entrylist entrylist = new Entrylist(); -// EntryOverview entryOverview = new EntryOverview(); -// entryOverview.mediaAssistantId = "00044741"; -// entryOverview.entryStateId = 6; -// entryOverview.mediaTypeId = 2; -// EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview }; -// entrylist.entries = entryOverviews; + // XXX + Entrylist entrylist = new Entrylist(); + EntryOverview entryOverview = new EntryOverview(); + entryOverview.mediaAssistantId = "00044741"; + entryOverview.entryStateId = 6; + entryOverview.mediaTypeId = 2; + EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview }; + entrylist.entries = entryOverviews; for (int i = 0; i < entrylist.entries.length; i++) { if (!downloadFiles) { // only create a list with all IDs try { - log.info("Get " + i + "/" + entrylist.entries.length); + log.info("Get " + (i+1) + "/" + entrylist.entries.length); TimeUnit.SECONDS.sleep(2); // XXX int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 12701d7..c85870c 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -70,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner { static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; // XXX - static final String fileCache = "E:\\_TEST\\"; + static final String fileCache = "E:\\"; static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; @@ -602,6 +602,8 @@ public class MamMigrationSelective implements CommandLineRunner { File file = getFileForEntry(download, entry.id); if (file != null) filesForUpload.add(file); + else + log.error("File " + download.fileName + " not available"); } } @@ -725,14 +727,20 @@ public class MamMigrationSelective implements CommandLineRunner { // check whether file is in cache, download if it's not there if (!file.exists()) { - log.debug("Downloading missing file " + filenameToUse + " for entry " + id); - URL url = new URL(download.href); - HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); - httpConn.setRequestProperty("Authorization", mamAuth); - ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream()); - FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - fos.close(); + try { + log.debug("Downloading missing file " + filenameToUse + " for entry " + id + " at " + download.href); + URL url = new URL(download.href); + HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); + httpConn.setRequestProperty("Authorization", "Basic " + mamAuth); + ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream()); + FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + } + catch (IOException ioe) { + log.error("Could not download file " + filenameToUse + " for entry " + id + ": " + ioe.getMessage()); + return null; + } } else { log.debug("Using existing file " + filenameToUse + " for entry " + id);