gexce
813 lines
32 KiB
Java
813 lines
32 KiB
Java
package com.bayer.edam.migration;
|
|
|
|
import java.io.File;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.URI;
|
|
import java.net.URL;
|
|
import java.nio.channels.Channels;
|
|
import java.nio.channels.ReadableByteChannel;
|
|
import java.nio.charset.Charset;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Collections;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.apache.commons.io.FileUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.boot.CommandLineRunner;
|
|
import org.springframework.boot.SpringApplication;
|
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
|
import org.springframework.context.annotation.PropertySource;
|
|
import org.springframework.core.io.FileSystemResource;
|
|
import org.springframework.http.HttpEntity;
|
|
import org.springframework.http.HttpHeaders;
|
|
import org.springframework.http.HttpMethod;
|
|
import org.springframework.http.ResponseEntity;
|
|
import org.springframework.http.client.BufferingClientHttpRequestFactory;
|
|
import org.springframework.http.client.ClientHttpRequestInterceptor;
|
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
|
import org.springframework.util.LinkedMultiValueMap;
|
|
import org.springframework.util.MultiValueMap;
|
|
import org.springframework.web.client.HttpStatusCodeException;
|
|
import org.springframework.web.client.RestClientException;
|
|
import org.springframework.web.client.RestTemplate;
|
|
|
|
import com.bayer.edam.migration.adam.Campaign;
|
|
import com.bayer.edam.migration.adam.Classification;
|
|
import com.bayer.edam.migration.adam.ClassificationList;
|
|
import com.bayer.edam.migration.adam.EntityId;
|
|
import com.bayer.edam.migration.adam.Token;
|
|
import com.bayer.edam.migration.adam.UploadClassification;
|
|
import com.bayer.edam.migration.adam.UploadFileRecord;
|
|
import com.bayer.edam.migration.adam.UploadRecord;
|
|
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
|
|
import com.bayer.edam.migration.adam.UploadResult;
|
|
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
|
|
import com.bayer.edam.migration.mam.CategoryTree;
|
|
import com.bayer.edam.migration.mam.CategoryTree.Node;
|
|
import com.bayer.edam.migration.mam.Entry;
|
|
import com.bayer.edam.migration.mam.Entry.Download;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
import net.lingala.zip4j.core.ZipFile;
|
|
import net.lingala.zip4j.model.FileHeader;
|
|
|
|
@SpringBootApplication
|
|
@PropertySource("file:secret.properties")
|
|
public class MamMigrationSelective implements CommandLineRunner {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
|
|
|
|
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
|
|
|
|
// XXX
|
|
static final String fileCache = "E:\\_TEST\\";
|
|
|
|
static final String categoryJson = "resources/2017-10-19_categories.json";
|
|
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
|
|
static final String entryList = "resources/entrylist_FULL_2017-11-15.txt";
|
|
|
|
@Value("${mam.auth}")
|
|
private String mamAuth;
|
|
|
|
@Value("${adam.auth}")
|
|
private String adamAuth;
|
|
|
|
RestTemplate restTemplate, fileTransferTemplate;
|
|
HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
|
|
HttpEntity<String> adamHeadersEntity;
|
|
|
|
private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
|
|
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
|
|
private ObjectMapper jsonMapper = new ObjectMapper();
|
|
|
|
private HashMap<Long, Entry> entryMap;
|
|
private List<Classification> campaignYears;
|
|
private List<Campaign> campaigns = new LinkedList<Campaign>();
|
|
private CategoryTree mamCategoryTree;
|
|
|
|
static final String fieldSeparator = "|~|";
|
|
static final String lineSeparator = "\r\n";
|
|
|
|
private List<UploadReference> openUploads = new ArrayList<UploadReference>();
|
|
|
|
@Override
|
|
public void run(String... strings) throws Exception {
|
|
|
|
entryMap = readEntriesFromFile(entryList);
|
|
|
|
List<Long> entrySelection;
|
|
|
|
|
|
|
|
// XXX ----------------------------------------------------------------
|
|
// Option 1: Process all available entries
|
|
// entrySelection = new ArrayList<Long>(entryMap.keySet());
|
|
|
|
// Option 2: Process a fixed set of entry IDs
|
|
entrySelection = Arrays.asList(71725L, 71263L, 71048L, 70778L, 70746L, 70745L, 70440L, 70361L, 70340L, 70288L, 70161L, 70142L,
|
|
63924L, 63201L, 57641L, 56942L);
|
|
|
|
// Option 3: Read entry IDs from file
|
|
// entrySelection = new ArrayList<Long>();
|
|
// List<String> entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8"));
|
|
// for (String idString : entryIDsList) {
|
|
// entrySelection.add(Long.parseLong(idString));
|
|
// }
|
|
// --------------------------------------------------------------------
|
|
|
|
|
|
|
|
log.info("Read categories");
|
|
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
|
|
|
|
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
|
requestFactory.setBufferRequestBody(false);
|
|
fileTransferTemplate = new RestTemplate(requestFactory);
|
|
|
|
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
|
|
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
|
|
interceptors.add(new LoggingRequestInterceptor());
|
|
restTemplate.setInterceptors(interceptors);
|
|
|
|
adamHeaders = createAdamHeaders();
|
|
adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
|
|
adamUploadHeaders = createAdamHeaders();
|
|
adamUploadHeaders.set("Content-Type", "multipart/form-data");
|
|
adamUploadHeaders.set("Accept", "*/*");
|
|
mamHeaders = createMamHeaders();
|
|
|
|
log.info("Read campaigns");
|
|
campaignYears = restTemplate.exchange(Definitions.adamHost + "/classification/" + Definitions.campaignRoot + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
|
|
for (Classification yearClassification : campaignYears) {
|
|
List<Classification> campaignClassifications = restTemplate.exchange(Definitions.adamHost + "/classification/" + yearClassification.id + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
|
|
for (Classification campaignClassification : campaignClassifications) {
|
|
Campaign campaign = new Campaign(campaignClassification.name, campaignClassification.labels[0].value, yearClassification.name);
|
|
campaign.guid = campaignClassification.id;
|
|
campaigns.add(campaign);
|
|
}
|
|
}
|
|
|
|
int entryCounter = 1;
|
|
// HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
|
|
for (long id : entrySelection) {
|
|
|
|
|
|
|
|
// XXX
|
|
// // if needed start from a certain ID
|
|
// if (id > 44741)
|
|
// continue;
|
|
|
|
|
|
|
|
log.info("GET entry " + id + " (" + entryCounter++ + "/" + entrySelection.size() + ")");
|
|
|
|
// Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
|
|
Entry entry = entryMap.get(id);
|
|
|
|
log.debug("Name: " + entry.name.get("en"));
|
|
|
|
// check exclusion criteria
|
|
if ((entry.stateId == 5 || entry.stateId == 4) && entry.fileName == null && entry.categories.size() == 0) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + ", no file attached, no category set");
|
|
continue;
|
|
}
|
|
else if (entry.stateId == 7) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (LOCKED_BY_ADMIN)");
|
|
continue;
|
|
}
|
|
else if (entry.stateId == 8) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ASSIGNED_TO_INDEXATION)");
|
|
continue;
|
|
}
|
|
else if (entry.stateId == 10) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ARCHIVED)");
|
|
continue;
|
|
}
|
|
|
|
postToAdam(entry);
|
|
}
|
|
|
|
while (openUploads.size() > 0) {
|
|
log.debug("Sleeping...");
|
|
TimeUnit.SECONDS.sleep(10);
|
|
|
|
log.debug("Checking back on open uploads");
|
|
attachOpenUploads();
|
|
}
|
|
|
|
invalidateAdamToken();
|
|
}
|
|
|
|
void postToAdam(Entry entry) throws Exception {
|
|
UploadRecord record = new UploadRecord();
|
|
String entryJson = jsonMapper.writeValueAsString(entry);
|
|
List<String> keywords = new ArrayList<String>();
|
|
HashSet<String> keywordSet = new HashSet<String>();
|
|
HashSet<String> campaignGuids = new HashSet<String>();
|
|
|
|
// XXX
|
|
record.addClassification("Migration.Test");
|
|
|
|
// globally applicable classifications
|
|
record.addClassification("Migration");
|
|
record.addClassification("Country.Global");
|
|
record.addClassification("Agencies.Bayer AH");
|
|
record.addClassification("Usage.Restricted");
|
|
|
|
// media type
|
|
record.addClassification(Definitions.mediaTypes.get(entry.mediaType.id));
|
|
|
|
// process categories
|
|
if (entry.categories.size() == 0) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": No category assigned, cannot determine business unit and brand");
|
|
return;
|
|
}
|
|
|
|
if (entry.categories.size() > 1)
|
|
log.warn("Entry " + entry.id + " has " + entry.categories.size() + " categories assigned");
|
|
|
|
for (int categoryId : entry.categories) {
|
|
log.debug("Category: " + categoryId);
|
|
|
|
Node node = mamCategoryTree.getNode(categoryId);
|
|
log.debug("Node: " + node);
|
|
List<Node> path = new LinkedList<Node>();
|
|
|
|
Node tmpNode = node;
|
|
String currentBrand = null;
|
|
while (tmpNode.id != CategoryTree.ROOT_NODE_ID) {
|
|
if (tmpNode.id == 0) {
|
|
log.info("RESULT Skipping entry " + entry.id + ": Node outside of category tree (" + node.toString() + ")");
|
|
return;
|
|
}
|
|
|
|
path.add(0, tmpNode);
|
|
tmpNode = tmpNode.parent;
|
|
}
|
|
|
|
for (Node n : path) {
|
|
if (n.action.equals("not migrated")) {
|
|
log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")");
|
|
return;
|
|
}
|
|
|
|
if (n.action.equals("classify")) {
|
|
String[] parameters = n.parameter.split(",");
|
|
|
|
for (String p : parameters) {
|
|
|
|
// Status "Released" is weaker than the other statuses
|
|
if (p.startsWith("Status.")) {
|
|
if (p.equals("Status.Released") && record.hasClassificationStartingWith("Status.")) {
|
|
if (!record.hasClassificationStartingWith("Status.Released"))
|
|
log.warn("Classification " + p + " not added, different status already set");
|
|
continue;
|
|
}
|
|
else {
|
|
record.removeClassificationsStartingWith("Status.");
|
|
}
|
|
}
|
|
|
|
// Business unit "General" is weaker than the other business units
|
|
if (p.startsWith("BusinessUnit.")) {
|
|
if (p.equals("BusinessUnit.General") && record.hasClassificationStartingWith("BusinessUnit.")) {
|
|
if (!record.hasClassificationStartingWith("BusinessUnit.General"))
|
|
log.warn("Classification " + p + " not added, different business unit already set");
|
|
continue;
|
|
}
|
|
else {
|
|
record.removeClassificationsStartingWith("BusinessUnit.");
|
|
}
|
|
}
|
|
|
|
// Brands "General" and "Others" are weaker than the other brands
|
|
if (p.startsWith("Brand.")) {
|
|
// if ((p.equals("Brand.General") || p.equals("Brand.CAP.Others") || p.equals("Brand.FAP.Others"))
|
|
// && record.hasClassificationStartingWith("Brand.")) {
|
|
//
|
|
// // FIXME Check if the brand matches the business unit
|
|
// if (record.hasClassificationStartingWith("Brand.General")
|
|
// && !record.hasClassificationStartingWith("BusinessUnit.General")) {
|
|
// log.warn("Classification " + p + " not added, different brand already set");
|
|
// continue;
|
|
// }
|
|
// }
|
|
|
|
record.removeClassificationsStartingWith("Brand.");
|
|
currentBrand = p.substring(p.lastIndexOf('.'));
|
|
}
|
|
|
|
if (p.startsWith("MediaType.")) {
|
|
record.removeClassificationsStartingWith("MediaType.");
|
|
}
|
|
|
|
record.addClassification(p);
|
|
}
|
|
}
|
|
else if (n.action.equals("brand")) {
|
|
record.removeClassificationsStartingWith("Brand.");
|
|
String brandClassificationId = "Brand." + n.name.trim();
|
|
record.addClassification(brandClassificationId);
|
|
currentBrand = n.name.trim();
|
|
}
|
|
else if (n.action.equals("campaign")) {
|
|
String campaignName = n.name;
|
|
String campaignLabel = n.name;
|
|
String campaignYear;
|
|
if (n.parameter != null && !n.parameter.trim().equals("")) {
|
|
campaignYear = n.parameter.substring(1, 5);
|
|
if (currentBrand != null)
|
|
campaignLabel = currentBrand + " - " + campaignLabel + " " + n.parameter;
|
|
else
|
|
campaignLabel = campaignLabel + " " + n.parameter;
|
|
} else {
|
|
campaignYear = n.name.substring(0, 4);
|
|
if (currentBrand != null)
|
|
campaignLabel = currentBrand + " - " + campaignLabel;
|
|
}
|
|
|
|
try {
|
|
Integer.parseInt(campaignYear);
|
|
|
|
// check whether the year classification already exists
|
|
Classification yearClassification = null;
|
|
for (Classification c : campaignYears) {
|
|
if (c.name.equals(campaignYear)) {
|
|
yearClassification = c;
|
|
}
|
|
}
|
|
|
|
if (yearClassification == null) {
|
|
log.debug("Create campaign year " + campaignYear);
|
|
|
|
UploadClassification newYear = new UploadClassification(campaignYear, campaignYear);
|
|
|
|
HttpEntity<UploadClassification> httpEntity = new HttpEntity<UploadClassification>(newYear, adamHeaders);
|
|
try {
|
|
EntityId classificationId = restTemplate.exchange(Definitions.adamHost + "/classifications", HttpMethod.POST, httpEntity, EntityId.class).getBody();
|
|
newYear.guid = classificationId.id;
|
|
|
|
// retrieve the newly created classification back
|
|
yearClassification = restTemplate.exchange(Definitions.adamHost + "/classification/" + newYear.guid, HttpMethod.GET, adamHeadersEntity, Classification.class).getBody();
|
|
campaignYears.add(yearClassification);
|
|
|
|
log.debug("New campaign year " + campaignYear + " created: " + yearClassification.id);
|
|
} catch (HttpStatusCodeException hsce) {
|
|
log.error("Could not create campaign year '" + campaignYear + "' for entry " + entry.id + ": " + hsce.getStatusCode().toString());
|
|
log.error(hsce.getResponseBodyAsString());
|
|
hsce.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
// check whether campaign already exists
|
|
String campaignGuid = null;
|
|
for (Campaign c : campaigns) {
|
|
if (c.name.equals(campaignName)) {
|
|
campaignGuid = c.guid;
|
|
if (campaignGuids.add(campaignGuid)) {
|
|
log.debug("Add existing campaign " + campaignName);
|
|
record.addClassification(new WriteClassification(campaignGuid, 1));
|
|
}
|
|
}
|
|
}
|
|
|
|
if (campaignGuid == null) {
|
|
log.info("Create campaign " + campaignYear + "/" + campaignName);
|
|
|
|
Campaign campaign = new Campaign(campaignName, campaignLabel, campaignYear);
|
|
|
|
HttpEntity<Campaign> httpEntity = new HttpEntity<Campaign>(campaign, adamHeaders);
|
|
try {
|
|
EntityId campaignId = restTemplate.exchange(Definitions.adamHost + "/classifications", HttpMethod.POST, httpEntity, EntityId.class).getBody();
|
|
campaign.guid = campaignId.id;
|
|
campaigns.add(campaign);
|
|
if (campaignGuids.add(campaign.guid)) {
|
|
log.debug("Add new campaign " + campaign.name);
|
|
record.addClassification(new WriteClassification(campaign.guid, 1));
|
|
}
|
|
}
|
|
catch (HttpStatusCodeException hsce) {
|
|
log.error("Could not create campaign '" + campaignName + "' for entry " + entry.id + ": " + hsce.getStatusCode().toString());
|
|
log.error(hsce.getResponseBodyAsString());
|
|
hsce.printStackTrace();
|
|
}
|
|
}
|
|
} catch (NumberFormatException nfe) {
|
|
log.error("Could not parse year from string " + campaignYear);
|
|
}
|
|
}
|
|
else if (n.action.equals("keyword")) {
|
|
if (keywordSet.add(n.name.trim().toLowerCase())) {
|
|
log.debug("Adding keyword " + n.name.trim());
|
|
keywords.add(n.name.trim());
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
// Set Status.Draft for entries with stateId == 5 (Created) or 4 (Assigned)
|
|
if (entry.stateId == 4 || entry.stateId == 5) {
|
|
log.debug("Entry has status " + entry.stateId + ", setting record status to 'Draft'");
|
|
record.removeClassification("Status.Released");
|
|
record.addClassification("Status.Draft");
|
|
}
|
|
|
|
// Fields
|
|
if (entry.name.get("en") != null) {
|
|
record.addField("byr_Title", entry.name.get("en"));
|
|
}
|
|
else {
|
|
log.warn("Entry " + entry.id + ": No title");
|
|
record.addField("byr_Title", "No title");
|
|
}
|
|
|
|
if (entry.creationDate != null)
|
|
record.addField("byr_ah_releaseDate", isoDate.format(entry.creationDate));
|
|
|
|
if (entry.metaData.get("photographer") != null)
|
|
record.addField("byr_ah_photographer", entry.metaData.get("photographer"));
|
|
|
|
// Project lead
|
|
if (entry.user != null && entry.user.get("id") != null) {
|
|
if (Definitions.users.get(entry.user.get("id")) != null) {
|
|
record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.users.get(entry.user.get("id"))));
|
|
record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.users.get(entry.user.get("id"))));
|
|
}
|
|
else {
|
|
log.warn("Entry " + entry.id + ": MAM user " + entry.user.get("id") + " not available, assigning default user");
|
|
record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
|
|
record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
|
|
}
|
|
}
|
|
else {
|
|
log.warn("Entry " + entry.id + ": No user, assigning default user");
|
|
record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
|
|
record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
|
|
}
|
|
|
|
// Description / Additional Information
|
|
String description = entry.description.get("en");
|
|
if (description == null)
|
|
description = "";
|
|
else
|
|
description += "\n\n";
|
|
|
|
if (entry.comment.get("en") != null)
|
|
description += "Comment: " + entry.comment.get("en") + "\n\n";
|
|
|
|
String migrationDescription = "Migrated from Media Assistant ID " + entry.id;
|
|
if (entry.creationDate != null)
|
|
migrationDescription += "\nDate of Issue: " + stringDate.format(entry.creationDate);
|
|
|
|
if (entry.categories.size() == 1)
|
|
migrationDescription += "\nCategory:";
|
|
else
|
|
migrationDescription += "\nCategories:";
|
|
for (int categoryId : entry.categories) {
|
|
migrationDescription += "\n" + mamCategoryTree.getNode(categoryId).toString();
|
|
}
|
|
|
|
|
|
description += migrationDescription;
|
|
record.addField("byr_ah_description", description);
|
|
|
|
// Keywords
|
|
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
|
|
List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
|
|
|
|
for (String s : mamKeywords) {
|
|
if (keywordSet.add(s.trim().toLowerCase())) {
|
|
keywords.add(s.trim());
|
|
}
|
|
|
|
// if the keyword is four digits - dash - four digits, it's a GPC number
|
|
if (s.matches("\\b\\d{4}-\\d{4}\\b"))
|
|
record.addField("byr_ah_approvalNumber", s);
|
|
}
|
|
}
|
|
|
|
if (keywords.size() > 0) {
|
|
record.addField("byr_ah_keywords_tl", keywords);
|
|
}
|
|
|
|
// License Information
|
|
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
|
|
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
|
|
|
|
if (entry.copyright != null && entry.copyright.organization != null) {
|
|
String licenseDescription = "";
|
|
if (entry.copyright.use != null)
|
|
licenseDescription += entry.copyright.use;
|
|
record.addField("byr_ah_licenseComments", licenseDescription);
|
|
|
|
// if license holder is not "Bayer Animal Health GmbH" put on review list
|
|
if (!entry.copyright.organization.get("id").equals("3740"))
|
|
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
|
|
}
|
|
else {
|
|
// if license information is missing put on review list
|
|
log.warn("Entry " + entry.id + ": Copyright information missing");
|
|
}
|
|
|
|
if (entry.gpc != null)
|
|
record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber);
|
|
|
|
// Create record
|
|
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
|
|
try {
|
|
|
|
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
|
|
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
|
|
|
|
uploadFilesForEntry(entry, location);
|
|
|
|
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
|
|
}
|
|
catch (RestClientException rce) {
|
|
log.error("RESULT Could not create record for entry " + entry.id + ": " + rce.getMessage());
|
|
rce.printStackTrace();
|
|
}
|
|
}
|
|
|
|
|
|
private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
|
|
HashSet<String> hrefsForUpload = new HashSet<String>();
|
|
ArrayList<File> filesForUpload = new ArrayList<File>();
|
|
|
|
// Select download: Download original only. If original is not available, download largest file
|
|
Download selectedDownload = null;
|
|
for (Download download : entry.downloadList) {
|
|
if (download.entryFileVariation.equals("ORIGINAL")) {
|
|
selectedDownload = download;
|
|
break;
|
|
}
|
|
|
|
if (selectedDownload == null) {
|
|
selectedDownload = download;
|
|
}
|
|
else if (selectedDownload.size < download.size) {
|
|
selectedDownload = download;
|
|
}
|
|
}
|
|
|
|
if (selectedDownload != null) {
|
|
|
|
File masterFile = getFileForEntry(selectedDownload, entry.id);
|
|
|
|
if (masterFile != null) {
|
|
// if selected download is a zip that contains a single file, upload extracted file as master
|
|
if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) {
|
|
ZipFile zipFile = new ZipFile(masterFile);
|
|
@SuppressWarnings("unchecked")
|
|
List<FileHeader> fileHeaders = zipFile.getFileHeaders();
|
|
|
|
if (fileHeaders.size() == 1) {
|
|
String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
|
|
|
|
if (!(new File(extractedMasterFilePath).exists())) {
|
|
log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName());
|
|
zipFile.extractAll(fileCache + entry.id + "\\");
|
|
}
|
|
log.info("Using extracted master file " + extractedMasterFilePath);
|
|
masterFile = new File(extractedMasterFilePath);
|
|
}
|
|
}
|
|
|
|
filesForUpload.add(masterFile);
|
|
hrefsForUpload.add(selectedDownload.href);
|
|
}
|
|
else {
|
|
log.error("Master file " + selectedDownload.fileName + " not available");
|
|
}
|
|
}
|
|
else {
|
|
log.warn("Could not identify master file for entry " + entry.id);
|
|
}
|
|
|
|
for (Download download : entry.downloadListPublicFolder) {
|
|
if (hrefsForUpload.add(download.href)) {
|
|
File file = getFileForEntry(download, entry.id);
|
|
if (file != null)
|
|
filesForUpload.add(file);
|
|
}
|
|
}
|
|
|
|
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
|
|
boolean isMaster = true;
|
|
for (File file : filesForUpload) {
|
|
if (file.exists()) {
|
|
if (file.length() < 4000000000L) {
|
|
log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
|
|
|
|
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
|
|
parameters.add("file", new FileSystemResource(file));
|
|
|
|
try {
|
|
UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
|
|
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
|
|
|
|
if (result.token != null) {
|
|
log.debug("Storing upload token " + result.token + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
|
|
UploadReference reference = new UploadReference();
|
|
reference.targetRecord = recordLocation;
|
|
reference.isMaster = isMaster;
|
|
reference.token = result.token;
|
|
reference.cachePath = file.getAbsolutePath();
|
|
openUploads.add(reference);
|
|
}
|
|
else if (result.uri != null) {
|
|
log.debug("Storing upload reference " + result.uri + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
|
|
UploadReference reference = new UploadReference();
|
|
reference.targetRecord = recordLocation;
|
|
reference.isMaster = isMaster;
|
|
reference.uri = result.uri;
|
|
reference.cachePath = file.getAbsolutePath();
|
|
openUploads.add(reference);
|
|
}
|
|
else {
|
|
log.error("No token/URI after upload!");
|
|
}
|
|
|
|
}
|
|
catch (RestClientException rce) {
|
|
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id + ": " + rce.getMessage());
|
|
rce.printStackTrace();
|
|
}
|
|
|
|
}
|
|
else {
|
|
// TODO
|
|
log.debug("File too large, could not upload " + file.getName() + " for " + recordLocation);
|
|
}
|
|
}
|
|
else {
|
|
log.error("Could not find file on disk " + file.getAbsolutePath());
|
|
}
|
|
|
|
isMaster = false;
|
|
}
|
|
|
|
// attach uploaded files to records where possible
|
|
attachOpenUploads();
|
|
|
|
}
|
|
|
|
private void attachOpenUploads() {
|
|
List<UploadReference> processedReferences = new ArrayList<UploadReference>();
|
|
|
|
for (UploadReference reference : openUploads) {
|
|
if (reference.token != null) {
|
|
UploadFileRecord fileRecord = new UploadFileRecord();
|
|
fileRecord.addFile(reference.token, reference.isMaster);
|
|
|
|
log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord + " via " + reference.token + " from " + reference.uri);
|
|
HttpEntity<UploadFileRecord> httpEntity = new HttpEntity<UploadFileRecord>(fileRecord, adamHeaders);
|
|
|
|
try {
|
|
ResponseEntity<Void> response = restTemplate.exchange(Definitions.adamHost + reference.targetRecord, HttpMethod.PUT, httpEntity, Void.class);
|
|
log.debug("ATTACH RESULT Attached " + reference.cachePath + " (" + reference.isMaster + ") to " + reference.targetRecord);
|
|
}
|
|
catch (RestClientException rce) {
|
|
log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath + " (" + reference.isMaster + ") for " + reference.targetRecord + ": " + rce.getMessage());
|
|
rce.printStackTrace();
|
|
}
|
|
|
|
processedReferences.add(reference);
|
|
}
|
|
else {
|
|
log.debug("Checking status of " + reference.cachePath + " for " + reference.uri);
|
|
try {
|
|
UploadResult result = restTemplate.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class).getBody();
|
|
log.debug("Result: " + result);
|
|
|
|
if (result.token != null) {
|
|
log.debug("Storing upload token " + result.token);
|
|
reference.token = result.token;
|
|
}
|
|
else if (result.status != null) {
|
|
if (result.status.equals("Error")) {
|
|
log.error("System could not process file " + reference.cachePath + " for " + reference.targetRecord);
|
|
processedReferences.add(reference);
|
|
}
|
|
}
|
|
}
|
|
catch (RestClientException rce) {
|
|
log.error("Failed to check status of " + reference.cachePath + " for " + reference.uri);
|
|
}
|
|
}
|
|
}
|
|
|
|
openUploads.removeAll(processedReferences);
|
|
log.debug(openUploads.size() + " open uploads remaining");
|
|
}
|
|
|
|
private File getFileForEntry(Download download, Long id) throws IOException {
|
|
String filenameToUse;
|
|
if (download.fileNameOnDisc != null)
|
|
filenameToUse = download.fileNameOnDisc;
|
|
else
|
|
filenameToUse = download.fileName;
|
|
|
|
File file = new File(fileCache + id + "\\" + filenameToUse);
|
|
|
|
// check whether file is in cache, download if it's not there
|
|
if (!file.exists()) {
|
|
log.debug("Downloading missing file " + filenameToUse + " for entry " + id);
|
|
URL url = new URL(download.href);
|
|
HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
|
|
httpConn.setRequestProperty("Authorization", mamAuth);
|
|
ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream());
|
|
FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
|
|
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
|
fos.close();
|
|
}
|
|
else {
|
|
log.debug("Using existing file " + filenameToUse + " for entry " + id);
|
|
}
|
|
|
|
return file;
|
|
}
|
|
|
|
private HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
|
|
File entryListFile = new File(filePath);
|
|
log.debug("Reading entries from " + entryListFile.getAbsolutePath());
|
|
List<String> entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8"));
|
|
HashMap<Long, Entry> entryMap = new HashMap<Long, Entry>();
|
|
|
|
for (String entryLine : entryList) {
|
|
|
|
String entryIdString = entryLine.substring(0, entryLine.indexOf(fieldSeparator));
|
|
Long entryId = Long.parseLong(entryIdString);
|
|
|
|
String entryJson = entryLine.substring(entryLine.indexOf(fieldSeparator) + 3);
|
|
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
|
|
|
|
entryMap.put(entryId, entry);
|
|
}
|
|
|
|
return entryMap;
|
|
}
|
|
|
|
private HttpHeaders createMamHeaders() {
|
|
HttpHeaders headers = new HttpHeaders();
|
|
headers.add("Accept", "application/json");
|
|
headers.add("Authorization", "Basic " + mamAuth);
|
|
headers.add("Content-Type", "application/json");
|
|
headers.add("From", "migration@bayer.com");
|
|
|
|
return headers;
|
|
}
|
|
|
|
private HttpHeaders createAdamHeaders() {
|
|
HttpHeaders headers = new HttpHeaders();
|
|
headers.add("Registration", "ADAMBAYER");
|
|
headers.add("API-VERSION", "1");
|
|
headers.add("Accept", "application/hal+json");
|
|
headers.add("Content-Type", "application/json");
|
|
headers.add("Authorization", "Basic " + adamAuth);
|
|
headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
|
|
HttpEntity<String> entity = new HttpEntity<String>("", headers);
|
|
Token token = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody();
|
|
|
|
// replace auth header
|
|
headers.remove("Authorization");
|
|
headers.add("Authorization", "Token " + token.getToken());
|
|
|
|
// replace accept-charset header (in order to declutter log)
|
|
headers.add("Accept-Charset", "us-ascii, utf-16, utf-8");
|
|
|
|
return headers;
|
|
}
|
|
|
|
private void invalidateAdamToken() {
|
|
ResponseEntity<String> response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
|
|
log.info("Logged out from adam API: " + response.toString());
|
|
}
|
|
|
|
/*
|
|
* // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new
|
|
* SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
|
*
|
|
* public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
|
|
* (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; }
|
|
*/
|
|
|
|
public static void main(String args[]) {
|
|
SpringApplication.run(MamMigrationSelective.class, args);
|
|
}
|
|
|
|
}
|