diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index e6d7c6a..83c6a46 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -2,4 +2,4 @@ logging.level.com.bayer=DEBUG #logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application_2017-11-14_QA_fallback.log +logging.file=application.log diff --git a/AH MAM Migration/resources/directory_listing.7z b/AH MAM Migration/resources/directory_listing.7z deleted file mode 100644 index 50f8ada..0000000 Binary files a/AH MAM Migration/resources/directory_listing.7z and /dev/null differ diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java index 629e7c8..fe98867 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java @@ -27,15 +27,12 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.client.BufferingClientHttpRequestFactory; -import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; -import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor; import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entrylist; -import com.bayer.edam.migration.mam.Entrylist.EntryOverview; import com.fasterxml.jackson.databind.ObjectMapper; //@SpringBootApplication @@ -47,11 +44,11 @@ public class EntryDownloader implements CommandLineRunner { /** * If true, files are downloaded. If false, only the IDs are written into the outputfile. */ - private final boolean downloadFiles = true; + private final boolean downloadFiles = false; private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\"; - File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-09.txt"); + File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\entrylist_FULL_2017-11-15.txt"); @Value("${mam.auth}") @@ -90,9 +87,9 @@ public class EntryDownloader implements CommandLineRunner { public void run(String... strings) { restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory())); - List interceptors = new ArrayList(); - interceptors.add(new LoggingRequestInterceptor()); - restTemplate.setInterceptors(interceptors); +// List interceptors = new ArrayList(); +// interceptors.add(new LoggingRequestInterceptor()); +// restTemplate.setInterceptors(interceptors); HttpHeaders headers = new HttpHeaders(); headers.add("Accept", "application/json"); @@ -110,10 +107,7 @@ public class EntryDownloader implements CommandLineRunner { lastPage = entrylist.lastPage; log.debug("processing page " + pageNum + " of " + lastPage); - - - - // XXX +// // XXX // Entrylist entrylist = new Entrylist(); // EntryOverview entryOverview = new EntryOverview(); // entryOverview.mediaAssistantId = "00044741"; @@ -122,17 +116,13 @@ public class EntryDownloader implements CommandLineRunner { // EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview }; // entrylist.entries = entryOverviews; - - - - for (int i = 0; i < entrylist.entries.length; i++) { if (!downloadFiles) { // only create a list with all IDs try { log.info("Get " + i + "/" + entrylist.entries.length); - TimeUnit.SECONDS.sleep(1); // XXX + TimeUnit.SECONDS.sleep(2); // XXX int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); String entryJson = jsonMapper.writeValueAsString(entry); @@ -146,10 +136,6 @@ public class EntryDownloader implements CommandLineRunner { Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId); - // XXX -// if (entryId != 44741) -// continue; - File targetDir = new File(basePath + entryId); if (targetDir.exists()) { log.info("Skip " + entryId + ", target directory already present"); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 0718512..634bd29 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -1,13 +1,19 @@ package com.bayer.edam.migration; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -62,16 +68,12 @@ public class MamMigrationSelective implements CommandLineRunner { private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class); static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; - private List entrySelection = new ArrayList(Arrays.asList( 46061/*71725, 71263, 71048, 70778, 70746, 70745, 70440, 70361, 70340, 70288, 70161, 70142, 63924, 63201, 57641, 56942*/ )); static final String fileCache = "E:\\"; static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; - - // XXX -// static final String entryListPath = "resources/entrylist_2017-11-09.txt"; - static final String entryListPath = "resources/lottery-min.txt"; + static final String entryList = "resources/entrylist_FULL_2017-11-15.txt"; @Value("${mam.auth}") private String mamAuth; @@ -87,6 +89,7 @@ public class MamMigrationSelective implements CommandLineRunner { private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); private ObjectMapper jsonMapper = new ObjectMapper(); + private HashMap entryMap; private List campaignYears; private List campaigns = new LinkedList(); private CategoryTree mamCategoryTree; @@ -99,8 +102,25 @@ public class MamMigrationSelective implements CommandLineRunner { @Override public void run(String... strings) throws Exception { + entryMap = readEntriesFromFile(entryList); + + List entrySelection; + // XXX -// readEntryIDsFromFile(); +// entrySelection = new ArrayList(entryMap.keySet()); + +// // XXX + entrySelection = Arrays.asList(71725L, 71263L, 71048L, 70778L, 70746L, 70745L, 70440L, 70361L, 70340L, 70288L, 70161L, 70142L, + 63924L, 63201L, 57641L, 56942L); + + // XXX +// entrySelection = new ArrayList(); +// List entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8")); +// for (String idString : entryIDsList) { +// entrySelection.add(Long.parseLong(idString)); +// } + + log.info("Read categories"); mamCategoryTree = new CategoryTree(categoryJson, categoryMapping); @@ -134,7 +154,7 @@ public class MamMigrationSelective implements CommandLineRunner { int entryCounter = 1; HttpEntity entity = new HttpEntity("", mamHeaders); - for (int id : entrySelection) { + for (long id : entrySelection) { // XXX // // if needed start from a certain ID @@ -145,7 +165,8 @@ public class MamMigrationSelective implements CommandLineRunner { log.info("GET entry " + id + " (" + entryCounter++ + ")"); - Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); +// Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); + Entry entry = entryMap.get(id); log.debug("Name: " + entry.name.get("en")); @@ -504,7 +525,8 @@ public class MamMigrationSelective implements CommandLineRunner { URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); log.info("RESULT Entry " + entry.id + " migrated to record " + location); -// uploadFilesForEntry(entry, location); + // XXX + uploadFilesForEntry(entry, location); log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); } @@ -697,16 +719,14 @@ public class MamMigrationSelective implements CommandLineRunner { // check whether file is in cache, download if it's not there if (!file.exists()) { - log.error("Missing file " + filenameToUse + " for entry " + id); - return null; - -// FIXME -// log.debug("Downloading missing file " + filenameToUse + " for entry " + id); -// URL fileLink = new URL(download.href); -// ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream()); -// FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); -// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); -// fos.close(); + log.debug("Downloading missing file " + filenameToUse + " for entry " + id); + URL url = new URL(download.href); + HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); + httpConn.setRequestProperty("Authorization", mamAuth); + ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream()); + FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); } else { log.debug("Using existing file " + filenameToUse + " for entry " + id); @@ -715,17 +735,24 @@ public class MamMigrationSelective implements CommandLineRunner { return file; } - private void readEntryIDsFromFile() throws IOException { - File entryIdListFile = new File(entryListPath); - log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath()); - List entryIdList = FileUtils.readLines(entryIdListFile, Charset.forName("utf-8")); + private HashMap readEntriesFromFile(String filePath) throws IOException { + File entryListFile = new File(filePath); + log.debug("Reading entries from " + entryListFile.getAbsolutePath()); + List entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8")); + HashMap entryMap = new HashMap(); - entrySelection = new ArrayList(); - - log.debug("Parsing " + entryIdList.size() + " IDs"); - for (String s : entryIdList) { - entrySelection.add(Integer.parseInt(s)); + for (String entryLine : entryList) { + + String entryIdString = entryLine.substring(0, entryLine.indexOf(fieldSeparator)); + Long entryId = Long.parseLong(entryIdString); + + String entryJson = entryLine.substring(entryLine.indexOf(fieldSeparator) + 3); + Entry entry = jsonMapper.readValue(entryJson, Entry.class); + + entryMap.put(entryId, entry); } + + return entryMap; } private HttpHeaders createMamHeaders() {