tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switched to cached JSON entries

Browse Source
This commit is contained in:
gexce
2017-11-16 11:06:45 +01:00
parent 78f40e7625
commit bb1a81ad3a
4 changed files with 63 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application_2017-11-14_QA_fallback.log
logging.file=application.log
AH MAM Migration/resources/directory_listing.7z
BIN
View File
Binary file not shown.
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+7 -21
View File
@@ -27,15 +27,12 @@ import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.bayer.edam.migration.mam.Entrylist.EntryOverview;
import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication
@@ -47,11 +44,11 @@ public class EntryDownloader implements CommandLineRunner {
/**
* If true, files are downloaded. If false, only the IDs are written into the outputfile.
*/
private final boolean downloadFiles = true;
private final boolean downloadFiles = false;
private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\";
File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-09.txt");
File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\entrylist_FULL_2017-11-15.txt");
@Value("${mam.auth}")
@@ -90,9 +87,9 @@ public class EntryDownloader implements CommandLineRunner {
public void run(String... strings) {
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
interceptors.add(new LoggingRequestInterceptor());
restTemplate.setInterceptors(interceptors);
// List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
// interceptors.add(new LoggingRequestInterceptor());
// restTemplate.setInterceptors(interceptors);
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json");
@@ -110,10 +107,7 @@ public class EntryDownloader implements CommandLineRunner {
lastPage = entrylist.lastPage;
log.debug("processing page " + pageNum + " of " + lastPage);
// XXX
// // XXX
// Entrylist entrylist = new Entrylist();
// EntryOverview entryOverview = new EntryOverview();
// entryOverview.mediaAssistantId = "00044741";
@@ -122,17 +116,13 @@ public class EntryDownloader implements CommandLineRunner {
// EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview };
// entrylist.entries = entryOverviews;
for (int i = 0; i < entrylist.entries.length; i++) {
if (!downloadFiles) {
// only create a list with all IDs
try {
log.info("Get " + i + "/" + entrylist.entries.length);
TimeUnit.SECONDS.sleep(1); // XXX
TimeUnit.SECONDS.sleep(2); // XXX
int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
String entryJson = jsonMapper.writeValueAsString(entry);
@@ -146,10 +136,6 @@ public class EntryDownloader implements CommandLineRunner {
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
// XXX
// if (entryId != 44741)
// continue;
File targetDir = new File(basePath + entryId);
if (targetDir.exists()) {
log.info("Skip " + entryId + ", target directory already present");
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+55 -28
View File
@@ -1,13 +1,19 @@
package com.bayer.edam.migration;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -62,16 +68,12 @@ public class MamMigrationSelective implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 46061/*71725, 71263, 71048, 70778, 70746, 70745, 70440, 70361, 70340, 70288, 70161, 70142, 63924, 63201, 57641, 56942*/ ));
static final String fileCache = "E:\\";
static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
// XXX
// static final String entryListPath = "resources/entrylist_2017-11-09.txt";
static final String entryListPath = "resources/lottery-min.txt";
static final String entryList = "resources/entrylist_FULL_2017-11-15.txt";
@Value("${mam.auth}")
private String mamAuth;
@@ -87,6 +89,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private ObjectMapper jsonMapper = new ObjectMapper();
private HashMap<Long, Entry> entryMap;
private List<Classification> campaignYears;
private List<Campaign> campaigns = new LinkedList<Campaign>();
private CategoryTree mamCategoryTree;
@@ -99,8 +102,25 @@ public class MamMigrationSelective implements CommandLineRunner {
@Override
public void run(String... strings) throws Exception {
entryMap = readEntriesFromFile(entryList);
List<Long> entrySelection;
// XXX
// readEntryIDsFromFile();
// entrySelection = new ArrayList<Long>(entryMap.keySet());
// // XXX
entrySelection = Arrays.asList(71725L, 71263L, 71048L, 70778L, 70746L, 70745L, 70440L, 70361L, 70340L, 70288L, 70161L, 70142L,
63924L, 63201L, 57641L, 56942L);
// XXX
// entrySelection = new ArrayList<Long>();
// List<String> entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8"));
// for (String idString : entryIDsList) {
// entrySelection.add(Long.parseLong(idString));
// }
log.info("Read categories");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
@@ -134,7 +154,7 @@ public class MamMigrationSelective implements CommandLineRunner {
int entryCounter = 1;
HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
for (int id : entrySelection) {
for (long id : entrySelection) {
// XXX
// // if needed start from a certain ID
@@ -145,7 +165,8 @@ public class MamMigrationSelective implements CommandLineRunner {
log.info("GET entry " + id + " (" + entryCounter++ + ")");
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
// Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
Entry entry = entryMap.get(id);
log.debug("Name: " + entry.name.get("en"));
@@ -504,7 +525,8 @@ public class MamMigrationSelective implements CommandLineRunner {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// uploadFilesForEntry(entry, location);
// XXX
uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
}
@@ -697,16 +719,14 @@ public class MamMigrationSelective implements CommandLineRunner {
// check whether file is in cache, download if it's not there
if (!file.exists()) {
log.error("Missing file " + filenameToUse + " for entry " + id);
return null;
// FIXME
// log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
// URL fileLink = new URL(download.href);
// ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
// FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// fos.close();
log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
URL url = new URL(download.href);
HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
httpConn.setRequestProperty("Authorization", mamAuth);
ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream());
FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
else {
log.debug("Using existing file " + filenameToUse + " for entry " + id);
@@ -715,17 +735,24 @@ public class MamMigrationSelective implements CommandLineRunner {
return file;
}
private void readEntryIDsFromFile() throws IOException {
File entryIdListFile = new File(entryListPath);
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());
List<String> entryIdList = FileUtils.readLines(entryIdListFile, Charset.forName("utf-8"));
private HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
File entryListFile = new File(filePath);
log.debug("Reading entries from " + entryListFile.getAbsolutePath());
List<String> entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8"));
HashMap<Long, Entry> entryMap = new HashMap<Long, Entry>();
entrySelection = new ArrayList<Integer>();
log.debug("Parsing " + entryIdList.size() + " IDs");
for (String s : entryIdList) {
entrySelection.add(Integer.parseInt(s));
for (String entryLine : entryList) {
String entryIdString = entryLine.substring(0, entryLine.indexOf(fieldSeparator));
Long entryId = Long.parseLong(entryIdString);
String entryJson = entryLine.substring(entryLine.indexOf(fieldSeparator) + 3);
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
entryMap.put(entryId, entry);
}
return entryMap;
}
private HttpHeaders createMamHeaders() {