tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EntryDownloader with files

Browse Source
This commit is contained in:
gexce
2017-09-22 16:22:04 +02:00
parent d04c957680
commit cee435f981
5 changed files with 158 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/questions.txt
+7 -2
View File
@@ -41,12 +41,17 @@
- Englisch OK
- Additional Info in einzelnen Zeilen [done]
- Comment und Description [done]
- Status: Liste mit Status != ACCEPTED an Claudia/Heike
- Status: Liste mit Status != ACCEPTED an Claudia/Heike [done]
Archiv ist nur andere Kategorie
- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess
- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin/Tilman klären Prozess
- Seite 2 als Preview
- Filename Preview nach Feld fileName setzen
-----------------------------------------------------
Status „Archived“ kann gelöscht werden bzw. muss nicht übernommen werden.
Heike prüft nochmal den Status „angelegt“ und gibt dann Rückmeldung.
Status „gesperrt“ kann gelöscht werden bzw. muss nicht übernommen werden.
Status „Zur Indexierung zugewiesen“ kann gelöscht bzw. muss nicht übernommen werden.
Status „Zum Upload zugewiesen“ bitte behalten.
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -1,4 +1,4 @@
# TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
logging.level.com.bayer=DEBUG
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=migration.log
logging.file=entryDownload.log
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+71 -12
View File
@@ -1,5 +1,13 @@
package com.bayer.edam.migration;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
@@ -17,7 +25,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication
@SpringBootApplication
@PropertySource("file:secret.properties")
public class EntryDownloader implements CommandLineRunner {
@@ -37,7 +45,7 @@ public class EntryDownloader implements CommandLineRunner {
@Override
public void run(String... strings) throws Exception {
public void run(String... strings) {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json");
headers.add("Authorization", "Basic " + mamAuth);
@@ -51,18 +59,69 @@ public class EntryDownloader implements CommandLineRunner {
lastPage = entrylist.lastPage;
for (int i = 0; i < entrylist.entries.length; i++) {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
long aggregatedFilesize = 0;
for (Download download : entry.downloadList) {
aggregatedFilesize += download.size;
try {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
// TODO extract, if zip --> postponed
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
long aggregatedFilesize = 0;
for (Download download : cleanedDownloads) {
aggregatedFilesize += download.size;
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
new File("E:\\" + entry.id).mkdir();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
FileUtils.copyURLToFile(new URL(download.href), new File("E:\\" + entry.id + "\\" + filenameToUse), 4000, 240000);
}
}
for (Download download : entry.downloadListPublicFolder) {
aggregatedFilesize += download.size;
catch (IOException ioe) {
log.error(ioe.toString());
ioe.printStackTrace();
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
}
}
}
AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java
+78 -39
View File
@@ -2,7 +2,9 @@ package com.bayer.edam.migration;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
@@ -10,6 +12,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
public class EntryProcessor {
@@ -30,62 +33,98 @@ public class EntryProcessor {
// skip header
iter.next();
HashSet<String> orgSet = new HashSet<String>();
int count = 0;
long aggregatedFilesize = 0;
while (iter.hasNext()) {
String line = iter.next();
line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
Entry entry = jsonMapper.readValue(line, Entry.class);
if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
String orgId = null;
if (entry.copyright != null && entry.copyright.organization != null) {
orgId = entry.copyright.organization.get("id");
orgSet.add(orgId);
List<Download> downloadSelection = new ArrayList<Download>();
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId + ", status: " + entry.stateId);
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
HashSet<String> filenames = new HashSet<String>();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
if (!filenames.add(filenameToUse)) {
log.error(entry.id + ": " + filenameToUse);
System.exit(1);
}
aggregatedFilesize += download.size;
}
/*
HashSet<String> filenames = new HashSet<String>();
HashSet<String> hrefs = new HashSet<String>();
for (Download download : downloadSelection) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
if (filenameToUse == null) {
log.error("ARGH " + entry.id);
System.exit(1);
}
if (filenames.add(filenameToUse)) {
aggregatedFilesize += download.size;
}
hrefs.add(download.href);
}
// for (Download download : entry.downloadListPublicFolder) {
//
// }
// boolean originalFound = false;
// long largest = 0;
// long originalSize = 0;
// for (Download download : entry.downloadList) {
// if (largest < download.size)
// largest = download.size;
//
// if (download.entryFileVariation.equals("ORIGINAL")) {
// originalFound = true;
// originalSize = download.size;
// }
// }
//
// if (!originalFound) {
// log.info(entry.id + " - no original - " + entry.stateId);
// count++;
// }
// else if (largest > originalSize) {
// log.info(entry.id + " - original is not largest file - " + entry.stateId);
// count++;
// }
if (filenames.size() < hrefs.size()) {
log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size());
for (String s : filenames)
log.error(s);
for (String s : hrefs)
log.error(s);
}
*/
}
for (String orgId : orgSet) {
log.info(orgId);
}
log.info("Count: " + count);
log.info("aggregatedFilesize: " + aggregatedFilesize);
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+1 -1
View File
@@ -88,7 +88,7 @@ public class MamMigrationSelective implements CommandLineRunner {
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
postToAdam(entry);
// postToAdam(entry);
}
// // delete all created records