tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed MAM download

Browse Source
This commit is contained in:
gexce
2017-11-17 11:28:46 +01:00
parent 9649cdb153
commit d5866ac48c
4 changed files with 39 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO #logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application_2017-11-16_QA_3_3.log logging.file=application_2017-11-17.log
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+1 -1
View File
@@ -49,7 +49,7 @@ public class Definitions {
users = new HashMap<String, String>(); users = new HashMap<String, String>();
// XXX QA values // XXX QA values
classifications.put("Migration.Test", new WriteClassification("de8edf2ed8d2405b87dea82d01062d50", 1)); classifications.put("Migration.Test", new WriteClassification("f7f19a55835e4936aa61a82d0116ae47", 1));
classifications.put("Migration", new WriteClassification("2bd24882-b7d8-4d30-8e6f-a7ef00eb33e7", 1)); classifications.put("Migration", new WriteClassification("2bd24882-b7d8-4d30-8e6f-a7ef00eb33e7", 1));
classifications.put("Migration.Logos", new WriteClassification("de96f8db1de44cc6841ba81c00b9db8e", 1)); classifications.put("Migration.Logos", new WriteClassification("de96f8db1de44cc6841ba81c00b9db8e", 1));
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+20 -17
View File
@@ -27,12 +27,15 @@ import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod; import org.springframework.http.HttpMethod;
import org.springframework.http.client.BufferingClientHttpRequestFactory; import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist; import com.bayer.edam.migration.mam.Entrylist;
import com.bayer.edam.migration.mam.Entrylist.EntryOverview;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication //@SpringBootApplication
@@ -44,11 +47,11 @@ public class EntryDownloader implements CommandLineRunner {
/** /**
* If true, files are downloaded. If false, only the IDs are written into the outputfile. * If true, files are downloaded. If false, only the IDs are written into the outputfile.
*/ */
private final boolean downloadFiles = false; private final boolean downloadFiles = true;
private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\"; private static final String basePath = "C:\\Temp\\Migration AH\\downtest\\";
File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\entrylist_FULL_2017-11-15.txt"); File outputfile = new File("C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\DOWNLOAD_LIST.txt");
@Value("${mam.auth}") @Value("${mam.auth}")
@@ -87,9 +90,9 @@ public class EntryDownloader implements CommandLineRunner {
public void run(String... strings) { public void run(String... strings) {
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory())); restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
// List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>(); List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
// interceptors.add(new LoggingRequestInterceptor()); interceptors.add(new LoggingRequestInterceptor());
// restTemplate.setInterceptors(interceptors); restTemplate.setInterceptors(interceptors);
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json"); headers.add("Accept", "application/json");
@@ -103,25 +106,25 @@ public class EntryDownloader implements CommandLineRunner {
int lastPage = 1; int lastPage = 1;
for (int pageNum = 1; pageNum <= lastPage; pageNum++) { for (int pageNum = 1; pageNum <= lastPage; pageNum++) {
Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody(); // Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody();
lastPage = entrylist.lastPage; // lastPage = entrylist.lastPage;
log.debug("processing page " + pageNum + " of " + lastPage); // log.debug("processing page " + pageNum + " of " + lastPage);
// // XXX // XXX
// Entrylist entrylist = new Entrylist(); Entrylist entrylist = new Entrylist();
// EntryOverview entryOverview = new EntryOverview(); EntryOverview entryOverview = new EntryOverview();
// entryOverview.mediaAssistantId = "00044741"; entryOverview.mediaAssistantId = "00044741";
// entryOverview.entryStateId = 6; entryOverview.entryStateId = 6;
// entryOverview.mediaTypeId = 2; entryOverview.mediaTypeId = 2;
// EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview }; EntryOverview[] entryOverviews = new EntryOverview[] { entryOverview };
// entrylist.entries = entryOverviews; entrylist.entries = entryOverviews;
for (int i = 0; i < entrylist.entries.length; i++) { for (int i = 0; i < entrylist.entries.length; i++) {
if (!downloadFiles) { if (!downloadFiles) {
// only create a list with all IDs // only create a list with all IDs
try { try {
log.info("Get " + i + "/" + entrylist.entries.length); log.info("Get " + (i+1) + "/" + entrylist.entries.length);
TimeUnit.SECONDS.sleep(2); // XXX TimeUnit.SECONDS.sleep(2); // XXX
int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId); int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+11 -3
View File
@@ -70,7 +70,7 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
// XXX // XXX
static final String fileCache = "E:\\_TEST\\"; static final String fileCache = "E:\\";
static final String categoryJson = "resources/2017-10-19_categories.json"; static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
@@ -602,6 +602,8 @@ public class MamMigrationSelective implements CommandLineRunner {
File file = getFileForEntry(download, entry.id); File file = getFileForEntry(download, entry.id);
if (file != null) if (file != null)
filesForUpload.add(file); filesForUpload.add(file);
else
log.error("File " + download.fileName + " not available");
} }
} }
@@ -725,15 +727,21 @@ public class MamMigrationSelective implements CommandLineRunner {
// check whether file is in cache, download if it's not there // check whether file is in cache, download if it's not there
if (!file.exists()) { if (!file.exists()) {
log.debug("Downloading missing file " + filenameToUse + " for entry " + id); try {
log.debug("Downloading missing file " + filenameToUse + " for entry " + id + " at " + download.href);
URL url = new URL(download.href); URL url = new URL(download.href);
HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
httpConn.setRequestProperty("Authorization", mamAuth); httpConn.setRequestProperty("Authorization", "Basic " + mamAuth);
ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream()); ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream());
FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close(); fos.close();
} }
catch (IOException ioe) {
log.error("Could not download file " + filenameToUse + " for entry " + id + ": " + ioe.getMessage());
return null;
}
}
else { else {
log.debug("Using existing file " + filenameToUse + " for entry " + id); log.debug("Using existing file " + filenameToUse + " for entry " + id);
} }