tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

90% of file upload, GPC number

Browse Source
This commit is contained in:
GEXCE
2017-11-08 15:42:27 +01:00
parent 4fdf6182b3
commit bb74e36db7
3 changed files with 147 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+100 -44
View File
@@ -1,13 +1,17 @@
package com.bayer.edam.migration;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -50,6 +54,9 @@ import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.model.FileHeader;
@SpringBootApplication
@PropertySource("file:secret.properties")
public class MamMigrationSelective implements CommandLineRunner {
@@ -59,6 +66,8 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 71263 /*72048, 60984 /*48147 /* 58443/*, 53787, 32427, 71725, 63080*/ )); // 72048, 60984
static final String fileCache = "E:\\";
static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
@@ -80,21 +89,17 @@ public class MamMigrationSelective implements CommandLineRunner {
private List<Campaign> campaigns = new LinkedList<Campaign>();
private CategoryTree mamCategoryTree;
static final SimpleDateFormat filenameDate = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
static final File entryLog = new File("entry_" + filenameDate.format(new Date()) + ".log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
private List<UploadReference> openUploads = new ArrayList<UploadReference>();
@Override
public void run(String... strings) throws Exception {
// XXX
// readEntryIDsFromFile();
log.info("Write log headers for " + entryLog.getAbsolutePath());
FileUtils.writeStringToFile(entryLog, "entryId" + fieldSeparator + "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
log.info("Read categories");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
@@ -407,7 +412,15 @@ public class MamMigrationSelective implements CommandLineRunner {
// Keywords
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
keywords.addAll(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
for (String s : mamKeywords) {
keywords.add(s);
// if the keyword is four digits - dash - four digits, it's a GPC number
if (s.matches("\\b\\d{4}-\\d{4}\\b"))
record.addField("byr_ah_approvalNumber", s);
}
}
if (keywords.size() > 0) {
@@ -440,16 +453,15 @@ public class MamMigrationSelective implements CommandLineRunner {
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
// URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
// log.info("RESULT Entry " + entry.id + " migrated to record " + location);
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX
uploadFilesForEntry(entry);
uploadFilesForEntry(entry, location);
// TODO process file upload errors
// FileUtils.writeStringToFile(entryLog, entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson + lineSeparator,
// Charset.forName("utf-8"), true);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
}
catch (HttpStatusCodeException hsce) {
log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
@@ -458,9 +470,9 @@ public class MamMigrationSelective implements CommandLineRunner {
}
private void uploadFilesForEntry(Entry entry) throws Exception {
List<Download> downloadSelection = new ArrayList<Download>();
private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
HashSet<String> hrefsForUpload = new HashSet<String>();
List<File> filesForUpload = new ArrayList<File>();
// Select download: Download original only. If original is not available, download largest file
Download selectedDownload = null;
@@ -478,62 +490,106 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
if (selectedDownload != null)
downloadSelection.add(selectedDownload);
if (selectedDownload != null) {
for (Download download : entry.downloadListPublicFolder) {
downloadSelection.add(download);
// if selected download is a zip that contains a single file, upload extracted file as master
if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) {
File masterFile = getFileForEntry(selectedDownload, entry.id);
ZipFile zipFile = new ZipFile(masterFile);
@SuppressWarnings("unchecked")
List<FileHeader> fileHeaders = zipFile.getFileHeaders();
if (fileHeaders.size() == 1) {
String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
if (!(new File(extractedMasterFilePath).exists())) {
log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName());
zipFile.extractAll(fileCache + entry.id + "\\");
}
log.info("Using extracted master file " + extractedMasterFilePath);
masterFile = new File(extractedMasterFilePath);
}
filesForUpload.add(masterFile);
hrefsForUpload.add(selectedDownload.href);
}
}
else {
log.warn("Could not identify master file for entry " + entry.id);
}
// Clean download selection
HashSet<String> hrefs = new HashSet<String>();
List<Download> cleanedDownloads = new ArrayList<Download>();
for (Download download : downloadSelection) {
if (hrefs.add(download.href)) {
cleanedDownloads.add(download);
for (Download download : entry.downloadListPublicFolder) {
if (hrefsForUpload.add(download.href)) {
filesForUpload.add(getFileForEntry(download, entry.id));
}
}
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
for (Download download : cleanedDownloads) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
String filepath = "E:\\" + entry.id + "\\" + filenameToUse;
File uploadFile = new File(filepath);
if (uploadFile.exists()) {
if (uploadFile.length() < 4000000000L) {
log.info("Uploading file " + filepath + ", size " + uploadFile.length());
for (File file : filesForUpload) {
if (file.exists()) {
if (file.length() < 4000000000L) {
log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(uploadFile));
parameters.add("file", new FileSystemResource(file));
UploadResult result = restTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
if (result.token != null) {
// TODO
log.debug("Token: " + result.token);
}
log.info("Upload result: " + result);
else if (result.uri != null) {
log.debug("Storing upload reference URI " + result.uri);
openUploads.add(new UploadReference(recordLocation, result.uri));
}
else {
log.error("No token/URI after upload!");
}
}
else {
log.debug("filesize " + uploadFile.length() + " > 4 GB!!!1!");
// TODO
log.debug("filesize " + file.length() + " > 4 GB!!!1!");
}
}
else {
log.error("Could not find file on disk " + filepath);
log.error("Could not find file on disk " + file.getAbsolutePath());
}
}
}
private File getFileForEntry(Download download, Long id) throws IOException {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
File file = new File(fileCache + id + "\\" + filenameToUse);
// check whether file is in cache, download if it's not there
if (!file.exists()) {
log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
URL fileLink = new URL(download.href);
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
else {
log.debug("Using existing file " + filenameToUse + " for entry " + id);
}
return file;
}
private void readEntryIDsFromFile() throws IOException {
File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());
AH MAM Migration/src/com/bayer/edam/migration/UploadReference.java
+15
View File
@@ -0,0 +1,15 @@
package com.bayer.edam.migration;
import java.net.URI;
public class UploadReference {
URI targetRecord;
String fileURI;
public UploadReference(URI record, String fileURI) {
this.targetRecord = record;
this.fileURI = fileURI;
}
}
AH MAM Migration/src/com/bayer/edam/migration/attic/ZipTest.java
+31
View File
@@ -0,0 +1,31 @@
package com.bayer.edam.migration.attic;
import java.util.List;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.model.FileHeader;
public class ZipTest {
public static void main(String[] args) {
try {
ZipFile zipFile = new ZipFile("C:\\Temp\\Migration AH\\ziptest\\folder.zip");
List<FileHeader> fileList = zipFile.getFileHeaders();
System.out.println(fileList.size());
for (FileHeader header : fileList) {
System.out.println(header.getFileName() + ", " + header.getUncompressedSize());
}
System.out.println(fileList.get(0).getFileName());
System.out.println("test".matches("\\d\\d\\d\\d-\\d\\d\\d\\d\\b"));
System.out.println("2121-5531".matches("\\b\\d{4}-\\d{4}\\b"));
} catch (Exception e) {
e.printStackTrace();
}
}
}