tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switched to cached JSON entries

Browse Source
This commit is contained in:
gexce
2017-11-16 11:06:45 +01:00
parent 78f40e7625
commit bb1a81ad3a
4 changed files with 63 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO #logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application_2017-11-14_QA_fallback.log logging.file=application.log
AH MAM Migration/resources/directory_listing.7z
BIN
View File
Binary file not shown.
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+7 -21
View File
@@ -27,15 +27,12 @@ import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod; import org.springframework.http.HttpMethod;
import org.springframework.http.client.BufferingClientHttpRequestFactory; import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist; import com.bayer.edam.migration.mam.Entrylist;
import com.bayer.edam.migration.mam.Entrylist.EntryOverview;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication //@SpringBootApplication
@@ -47,11 +44,11 @@ public class EntryDownloader implements CommandLineRunner {
/** /**
* If true, files are downloaded. If false, only the IDs are written into the outputfile. * If true, files are downloaded. If false, only the IDs are written into the outputfile.
*/ */
private final boolean downloadFiles = true; private final boolean downloadFiles = false;
private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\"; private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\";
File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-09.txt"); File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\entrylist_FULL_2017-11-15.txt");
@Value("${mam.auth}") @Value("${mam.auth}")
@@ -90,9 +87,9 @@ public class EntryDownloader implements CommandLineRunner {
public void run(String... strings) { public void run(String... strings) {
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory())); restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>(); // List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
interceptors.add(new LoggingRequestInterceptor()); // interceptors.add(new LoggingRequestInterceptor());
restTemplate.setInterceptors(interceptors); // restTemplate.setInterceptors(interceptors);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json"); headers.add("Accept", "application/json");
@@ -110,10 +107,7 @@ public class EntryDownloader implements CommandLineRunner {
lastPage = entrylist.lastPage; lastPage = entrylist.lastPage;
log.debug("processing page " + pageNum + " of " + lastPage); log.debug("processing page " + pageNum + " of " + lastPage);
// // XXX
// XXX
// Entrylist entrylist = new Entrylist(); // Entrylist entrylist = new Entrylist();
// EntryOverview entryOverview = new EntryOverview(); // EntryOverview entryOverview = new EntryOverview();
// entryOverview.mediaAssistantId = "00044741"; // entryOverview.mediaAssistantId = "00044741";
@@ -122,17 +116,13 @@ public class EntryDownloader implements CommandLineRunner {
// EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview }; // EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview };
// entrylist.entries = entryOverviews; // entrylist.entries = entryOverviews;
for (int i = 0; i < entrylist.entries.length; i++) { for (int i = 0; i < entrylist.entries.length; i++) {
if (!downloadFiles) { if (!downloadFiles) {
// only create a list with all IDs // only create a list with all IDs
try { try {
log.info("Get " + i + "/" + entrylist.entries.length); log.info("Get " + i + "/" + entrylist.entries.length);
TimeUnit.SECONDS.sleep(1); // XXX TimeUnit.SECONDS.sleep(2); // XXX
int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId); int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
String entryJson = jsonMapper.writeValueAsString(entry); String entryJson = jsonMapper.writeValueAsString(entry);
@@ -146,10 +136,6 @@ public class EntryDownloader implements CommandLineRunner {
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId); Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
// XXX
// if (entryId != 44741)
// continue;
File targetDir = new File(basePath + entryId); File targetDir = new File(basePath + entryId);
if (targetDir.exists()) { if (targetDir.exists()) {
log.info("Skip " + entryId + ", target directory already present"); log.info("Skip " + entryId + ", target directory already present");
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+54 -27
View File
@@ -1,13 +1,19 @@
package com.bayer.edam.migration; package com.bayer.edam.migration;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI; import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
@@ -62,16 +68,12 @@ public class MamMigrationSelective implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class); private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 46061/*71725, 71263, 71048, 70778, 70746, 70745, 70440, 70361, 70340, 70288, 70161, 70142, 63924, 63201, 57641, 56942*/ ));
static final String fileCache = "E:\\"; static final String fileCache = "E:\\";
static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
static final String entryList = "resources/entrylist_FULL_2017-11-15.txt";
// XXX
// static final String entryListPath = "resources/entrylist_2017-11-09.txt";
static final String entryListPath = "resources/lottery-min.txt";
@Value("${mam.auth}") @Value("${mam.auth}")
private String mamAuth; private String mamAuth;
@@ -87,6 +89,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private ObjectMapper jsonMapper = new ObjectMapper(); private ObjectMapper jsonMapper = new ObjectMapper();
private HashMap<Long, Entry> entryMap;
private List<Classification> campaignYears; private List<Classification> campaignYears;
private List<Campaign> campaigns = new LinkedList<Campaign>(); private List<Campaign> campaigns = new LinkedList<Campaign>();
private CategoryTree mamCategoryTree; private CategoryTree mamCategoryTree;
@@ -99,8 +102,25 @@ public class MamMigrationSelective implements CommandLineRunner {
@Override @Override
public void run(String... strings) throws Exception { public void run(String... strings) throws Exception {
entryMap = readEntriesFromFile(entryList);
List<Long> entrySelection;
// XXX // XXX
// readEntryIDsFromFile(); // entrySelection = new ArrayList<Long>(entryMap.keySet());
// // XXX
entrySelection = Arrays.asList(71725L, 71263L, 71048L, 70778L, 70746L, 70745L, 70440L, 70361L, 70340L, 70288L, 70161L, 70142L,
63924L, 63201L, 57641L, 56942L);
// XXX
// entrySelection = new ArrayList<Long>();
// List<String> entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8"));
// for (String idString : entryIDsList) {
// entrySelection.add(Long.parseLong(idString));
// }
log.info("Read categories"); log.info("Read categories");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping); mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
@@ -134,7 +154,7 @@ public class MamMigrationSelective implements CommandLineRunner {
int entryCounter = 1; int entryCounter = 1;
HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders); HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
for (int id : entrySelection) { for (long id : entrySelection) {
// XXX // XXX
// // if needed start from a certain ID // // if needed start from a certain ID
@@ -145,7 +165,8 @@ public class MamMigrationSelective implements CommandLineRunner {
log.info("GET entry " + id + " (" + entryCounter++ + ")"); log.info("GET entry " + id + " (" + entryCounter++ + ")");
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); // Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
Entry entry = entryMap.get(id);
log.debug("Name: " + entry.name.get("en")); log.debug("Name: " + entry.name.get("en"));
@@ -504,7 +525,8 @@ public class MamMigrationSelective implements CommandLineRunner {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location); log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// uploadFilesForEntry(entry, location); // XXX
uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
} }
@@ -697,16 +719,14 @@ public class MamMigrationSelective implements CommandLineRunner {
// check whether file is in cache, download if it's not there // check whether file is in cache, download if it's not there
if (!file.exists()) { if (!file.exists()) {
log.error("Missing file " + filenameToUse + " for entry " + id); log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
return null; URL url = new URL(download.href);
HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
// FIXME httpConn.setRequestProperty("Authorization", mamAuth);
// log.debug("Downloading missing file " + filenameToUse + " for entry " + id); ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream());
// URL fileLink = new URL(download.href); FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
// ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream()); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); fos.close();
// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// fos.close();
} }
else { else {
log.debug("Using existing file " + filenameToUse + " for entry " + id); log.debug("Using existing file " + filenameToUse + " for entry " + id);
@@ -715,17 +735,24 @@ public class MamMigrationSelective implements CommandLineRunner {
return file; return file;
} }
private void readEntryIDsFromFile() throws IOException { private HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
File entryIdListFile = new File(entryListPath); File entryListFile = new File(filePath);
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath()); log.debug("Reading entries from " + entryListFile.getAbsolutePath());
List<String> entryIdList = FileUtils.readLines(entryIdListFile, Charset.forName("utf-8")); List<String> entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8"));
HashMap<Long, Entry> entryMap = new HashMap<Long, Entry>();
entrySelection = new ArrayList<Integer>(); for (String entryLine : entryList) {
log.debug("Parsing " + entryIdList.size() + " IDs"); String entryIdString = entryLine.substring(0, entryLine.indexOf(fieldSeparator));
for (String s : entryIdList) { Long entryId = Long.parseLong(entryIdString);
entrySelection.add(Integer.parseInt(s));
String entryJson = entryLine.substring(entryLine.indexOf(fieldSeparator) + 3);
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
entryMap.put(entryId, entry);
} }
return entryMap;
} }
private HttpHeaders createMamHeaders() { private HttpHeaders createMamHeaders() {