tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EntryDownloader - download all entry IDs

Browse Source
This commit is contained in:
gexce
2017-10-25 14:59:13 +02:00
parent fae7943dc6
commit 9e62fa6583
5 changed files with 12753 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/Migration AH.7z
BIN
View File
Binary file not shown.
AH MAM Migration/resources/entryDownload_2017-10-16.log
+12654
View File
File diff suppressed because it is too large Load Diff
AH MAM Migration/resources/h2-bin.7z
BIN
View File
Binary file not shown.
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+93 -70
View File
@@ -8,10 +8,12 @@ import java.net.PasswordAuthentication;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
@@ -29,12 +31,20 @@ import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.fasterxml.jackson.databind.ObjectMapper;
//@SpringBootApplication
@SpringBootApplication
@PropertySource("file:secret.properties")
public class EntryDownloader implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(EntryDownloader.class);
/**
* If true, files are downloaded. If false, only the IDs are written into the outputfile.
*/
private final boolean downloadFiles = false;
File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-10-25.txt");
@Value("${mam.auth}")
private String mamAuth;
@@ -82,81 +92,94 @@ public class EntryDownloader implements CommandLineRunner {
for (int pageNum = 1; pageNum <= lastPage; pageNum++) {
Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody();
lastPage = entrylist.lastPage;
log.debug("processing page " + pageNum + " of " + lastPage);
for (int i = 0; i < entrylist.entries.length; i++) {
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
File targetDir = new File("E:\\" + entryId);
if (targetDir.exists()) {
log.info("Skip " + entryId + ", target directory already present");
continue;
}
try {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
// TODO extract, if zip --> postponed until upload
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
long aggregatedFilesize = 0;
for (Download download : cleanedDownloads) {
aggregatedFilesize += download.size;
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
targetDir.mkdir();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
URL fileLink = new URL(download.href);
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
if (!downloadFiles) {
// only create a list with all IDs
try {
FileUtils.writeStringToFile(outputfile, entrylist.entries[i].mediaAssistantId + "\n", Charset.forName("utf-8"), true);
} catch (IOException e) {
e.printStackTrace();
}
}
catch (IOException ioe) {
log.error(ioe.getMessage());
else {
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
File targetDir = new File("E:\\" + entryId);
if (targetDir.exists()) {
log.info("Skip " + entryId + ", target directory already present");
continue;
}
try {
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
List<Download> downloadSelection = new ArrayList<Download>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
for (Download download : entry.downloadList) {
if (download.entryFileVariation.equals("ORIGINAL")) {
selectedDownload = download;
break;
}
if (selectedDownload == null) {
selectedDownload = download;
}
else if (selectedDownload.size < download.size) {
selectedDownload = download;
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
// TODO extract, if zip --> postponed until upload
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
}
// clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
}
}
long aggregatedFilesize = 0;
for (Download download : cleanedDownloads) {
aggregatedFilesize += download.size;
}
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
targetDir.mkdir();
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
URL fileLink = new URL(download.href);
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
}
catch (IOException ioe) {
log.error(ioe.getMessage());
}
}
}
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+6 -16
View File
@@ -43,7 +43,7 @@ import com.bayer.edam.migration.mam.categories.CategoryTree;
import com.bayer.edam.migration.mam.categories.CategoryTree.Node;
import com.fasterxml.jackson.databind.ObjectMapper;
@SpringBootApplication
//@SpringBootApplication
@PropertySource("file:secret.properties")
public class MamMigrationSelective implements CommandLineRunner {
@@ -79,15 +79,14 @@ public class MamMigrationSelective implements CommandLineRunner {
private List<Classification> campaignYears;
private List<Campaign> campaigns = new LinkedList<Campaign>();
private List<String> locations = new ArrayList<String>();
private CategoryTree mamCategoryTree;
@Override
public void run(String... strings) throws Exception {
log.info("Write log headers");
log.info("Write log headers for " + entryLog.getAbsolutePath());
FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
log.info("Write log headers for " + problemLog.getAbsolutePath());
FileUtils.writeStringToFile(problemLog, "entryId" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
log.info("Read categories");
@@ -137,13 +136,6 @@ public class MamMigrationSelective implements CommandLineRunner {
postToAdam(entry);
}
// // delete all created records
// for (String location : locations) {
// ResponseEntity<String> response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE, adamHeadersEntity,
// String.class);
// log.info("Deleted " + location + ": " + response.toString());
// }
invalidateAdamToken();
}
@@ -331,14 +323,12 @@ public class MamMigrationSelective implements CommandLineRunner {
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
log.info("Created " + location);
log.info("Entry " + entry.id + " migrated to record " + location);
FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
locations.add(location.toString());
} catch (HttpStatusCodeException hsce) {
}
catch (HttpStatusCodeException hsce) {
log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
log.error(hsce.getResponseBodyAsString());
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);