tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

screw this, upload the selection only

Browse Source
This commit is contained in:
gexce
2017-11-10 15:05:03 +01:00
parent 86f51a97ca
commit 8cc74753bf
1 changed files with 38 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+38 -31
View File
@@ -1,12 +1,8 @@
package com.bayer.edam.migration; package com.bayer.edam.migration;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
@@ -74,8 +70,8 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
// XXX // XXX
static final String entryListPath = "resources/entrylist_2017-10-25.txt"; // static final String entryListPath = "resources/entrylist_2017-10-25.txt";
// static final String entryListPath = "resources/lottery.txt"; static final String entryListPath = "resources/lottery.txt";
@Value("${mam.auth}") @Value("${mam.auth}")
private String mamAuth; private String mamAuth;
@@ -191,7 +187,7 @@ public class MamMigrationSelective implements CommandLineRunner {
HashSet<String> campaignGuids = new HashSet<String>(); HashSet<String> campaignGuids = new HashSet<String>();
// XXX // XXX
// record.addClassification("Migration.Test"); record.addClassification("Migration.Test");
// globally applicable classifications // globally applicable classifications
record.addClassification("Migration"); record.addClassification("Migration");
@@ -523,7 +519,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception { private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
HashSet<String> hrefsForUpload = new HashSet<String>(); HashSet<String> hrefsForUpload = new HashSet<String>();
List<File> filesForUpload = new ArrayList<File>(); ArrayList<File> filesForUpload = new ArrayList<File>();
// Select download: Download original only. If original is not available, download largest file // Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null; Download selectedDownload = null;
@@ -545,26 +541,31 @@ public class MamMigrationSelective implements CommandLineRunner {
File masterFile = getFileForEntry(selectedDownload, entry.id); File masterFile = getFileForEntry(selectedDownload, entry.id);
// if selected download is a zip that contains a single file, upload extracted file as master if (masterFile != null) {
if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) { // if selected download is a zip that contains a single file, upload extracted file as master
ZipFile zipFile = new ZipFile(masterFile); if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) {
@SuppressWarnings("unchecked") ZipFile zipFile = new ZipFile(masterFile);
List<FileHeader> fileHeaders = zipFile.getFileHeaders(); @SuppressWarnings("unchecked")
List<FileHeader> fileHeaders = zipFile.getFileHeaders();
if (fileHeaders.size() == 1) {
String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
if (!(new File(extractedMasterFilePath).exists())) { if (fileHeaders.size() == 1) {
log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName()); String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
zipFile.extractAll(fileCache + entry.id + "\\");
if (!(new File(extractedMasterFilePath).exists())) {
log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName());
zipFile.extractAll(fileCache + entry.id + "\\");
}
log.info("Using extracted master file " + extractedMasterFilePath);
masterFile = new File(extractedMasterFilePath);
} }
log.info("Using extracted master file " + extractedMasterFilePath);
masterFile = new File(extractedMasterFilePath);
} }
filesForUpload.add(masterFile);
hrefsForUpload.add(selectedDownload.href);
}
else {
log.error("Master file " + selectedDownload.fileName + " not available");
} }
filesForUpload.add(masterFile);
hrefsForUpload.add(selectedDownload.href);
} }
else { else {
log.warn("Could not identify master file for entry " + entry.id); log.warn("Could not identify master file for entry " + entry.id);
@@ -572,7 +573,9 @@ public class MamMigrationSelective implements CommandLineRunner {
for (Download download : entry.downloadListPublicFolder) { for (Download download : entry.downloadListPublicFolder) {
if (hrefsForUpload.add(download.href)) { if (hrefsForUpload.add(download.href)) {
filesForUpload.add(getFileForEntry(download, entry.id)); File file = getFileForEntry(download, entry.id);
if (file != null)
filesForUpload.add(file);
} }
} }
@@ -691,12 +694,16 @@ public class MamMigrationSelective implements CommandLineRunner {
// check whether file is in cache, download if it's not there // check whether file is in cache, download if it's not there
if (!file.exists()) { if (!file.exists()) {
log.debug("Downloading missing file " + filenameToUse + " for entry " + id); log.error("Missing file " + filenameToUse + " for entry " + id);
URL fileLink = new URL(download.href); return null;
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse); // FIXME
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); // log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
fos.close(); // URL fileLink = new URL(download.href);
// ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
// FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
// fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
// fos.close();
} }
else { else {
log.debug("Using existing file " + filenameToUse + " for entry " + id); log.debug("Using existing file " + filenameToUse + " for entry " + id);