tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EntryDownloader with files

Browse Source
This commit is contained in:
gexce
2017-09-22 16:22:04 +02:00
parent d04c957680
commit cee435f981
5 changed files with 158 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+71 -12
View File
@@ -1,5 +1,13 @@
package com.bayer.edam.migration;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
@@ -17,7 +25,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication
@SpringBootApplication
@PropertySource("file:secret.properties")
public class EntryDownloader implements CommandLineRunner {
@@ -37,7 +45,7 @@ public class EntryDownloader implements CommandLineRunner {
@Override
public void run(String... strings) throws Exception {
public void run(String... strings) {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json");
headers.add("Authorization", "Basic " + mamAuth);
@@ -51,18 +59,69 @@ public class EntryDownloader implements CommandLineRunner {
lastPage = entrylist.lastPage;
for (int i = 0; i < entrylist.entries.length; i++) {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
long aggregatedFilesize = 0;
for (Download download : entry.downloadList) {
aggregatedFilesize += download.size;
try {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
// TODO extract, if zip --> postponed
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
long aggregatedFilesize = 0;
for (Download download : cleanedDownloads) {
aggregatedFilesize += download.size;
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
new File("E:\\" + entry.id).mkdir();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
FileUtils.copyURLToFile(new URL(download.href), new File("E:\\" + entry.id + "\\" + filenameToUse), 4000, 240000);
}
}
for (Download download : entry.downloadListPublicFolder) {
aggregatedFilesize += download.size;
catch (IOException ioe) {
log.error(ioe.toString());
ioe.printStackTrace();
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
}
}
}
AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java
+78 -39
View File
@@ -2,7 +2,9 @@ package com.bayer.edam.migration;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
@@ -10,6 +12,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
public class EntryProcessor {
@@ -30,62 +33,98 @@ public class EntryProcessor {
// skip header
iter.next();
HashSet<String> orgSet = new HashSet<String>();
int count = 0;
long aggregatedFilesize = 0;
while (iter.hasNext()) {
String line = iter.next();
line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
Entry entry = jsonMapper.readValue(line, Entry.class);
if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
String orgId = null;
if (entry.copyright != null && entry.copyright.organization != null) {
orgId = entry.copyright.organization.get("id");
orgSet.add(orgId);
List<Download> downloadSelection = new ArrayList<Download>();
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId + ", status: " + entry.stateId);
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
HashSet<String> filenames = new HashSet<String>();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
if (!filenames.add(filenameToUse)) {
log.error(entry.id + ": " + filenameToUse);
System.exit(1);
}
aggregatedFilesize += download.size;
}
/*
HashSet<String> filenames = new HashSet<String>();
HashSet<String> hrefs = new HashSet<String>();
for (Download download : downloadSelection) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
if (filenameToUse == null) {
log.error("ARGH " + entry.id);
System.exit(1);
}
if (filenames.add(filenameToUse)) {
aggregatedFilesize += download.size;
}
hrefs.add(download.href);
}
// for (Download download : entry.downloadListPublicFolder) {
//
// }
// boolean originalFound = false;
// long largest = 0;
// long originalSize = 0;
// for (Download download : entry.downloadList) {
// if (largest < download.size)
// largest = download.size;
//
// if (download.entryFileVariation.equals("ORIGINAL")) {
// originalFound = true;
// originalSize = download.size;
// }
// }
//
// if (!originalFound) {
// log.info(entry.id + " - no original - " + entry.stateId);
// count++;
// }
// else if (largest > originalSize) {
// log.info(entry.id + " - original is not largest file - " + entry.stateId);
// count++;
// }
if (filenames.size() < hrefs.size()) {
log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size());
for (String s : filenames)
log.error(s);
for (String s : hrefs)
log.error(s);
}
*/
}
for (String orgId : orgSet) {
log.info(orgId);
}
log.info("Count: " + count);
log.info("aggregatedFilesize: " + aggregatedFilesize);
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+1 -1
View File
@@ -88,7 +88,7 @@ public class MamMigrationSelective implements CommandLineRunner {
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
postToAdam(entry);
// postToAdam(entry);
}
// // delete all created records