tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

getting mature

Browse Source
This commit is contained in:
gexce
2017-10-26 15:44:21 +02:00
parent f104f3b3cc
commit 3db724bfba
7 changed files with 12814 additions and 12713 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+14 -2
View File
@@ -3,14 +3,16 @@ package com.bayer.edam.migration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.adam.UploadRecord.Field;
import com.bayer.edam.migration.adam.UploadRecord.LocalizedValue;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
public class Definitions {
public static final Map<String, WriteClassification> classifications;
public static final Map<String, String> classificationGuidToKeyMap;
public static final Map<String, String> fieldIDs;
public static final Map<Long, String> mediaTypes;
@@ -42,12 +44,15 @@ public class Definitions {
classifications.put("Status.Archived", new WriteClassification("4306204af75442fda297a7b4006b4a87", 1));
// /Bayer/Animal Health/Brands
classifications.put("Brand.General", new WriteClassification("92b4b27a09f2487dbba6a70f00f7baa3", 1));
classifications.put("Brand.Advantage", new WriteClassification("b3f04996de1644df9d08a6dd00cff70e", 1));
classifications.put("Brand.Advantix", new WriteClassification("dc758de7d91e44d6a768a6dd00cff7ad", 1));
classifications.put("Brand.Advocate", new WriteClassification("c7979fc30bb843e486bba6dd00cff835", 1));
classifications.put("Brand.Baytril", new WriteClassification("9c8a43adf3324ca6adb0a7d000f427f5", 1));
classifications.put("Brand.Bolfo/Bay-o-Pet", new WriteClassification("922a46fc3f6a44bfb025a7d000f428ac", 1));
classifications.put("Brand.CVBD", new WriteClassification("fc84b38665a44291a24ba70f00f7aa1f", 1));
classifications.put("Brand.Droncit", new WriteClassification("86251780660d4faf95cda7cb010c40fe", 1));
classifications.put("Brand.Drontal", new WriteClassification("ca18ffb9735e4f1691cfa6dd00cffb07", 1));
classifications.put("Brand.Procox", new WriteClassification("a64abb14f3fc461ab973a6dd00cffb8f", 1));
classifications.put("Brand.Profender", new WriteClassification("6be697c82ec34354b0c3a6dd00cffc04", 1));
@@ -67,8 +72,9 @@ public class Definitions {
classifications.put("Brand.Dairy Products", new WriteClassification("a54fcbb0dd4741059f95a6dd00d00289", 1));
classifications.put("Brand.Honey bee care", new WriteClassification("996f2d1656f44368a87ba6dd00d00353", 1));
classifications.put("Brand.Victrio", new WriteClassification("9d260d0e9e864ab3b3a4a6dd00d005f1", 1));
classifications.put("Brand.Polyvar", new WriteClassification("d41b837d516a49c0be05a81200e160cb", 1));
classifications.put("Brand.Zelnate", new WriteClassification("b3887552c9554d6991e4a6dd00d006ee", 1));
classifications.put("Brand.Droncit", new WriteClassification("86251780660d4faf95cda7cb010c40fe", 1));
classifications.put("Brand.FAP.Others", new WriteClassification("ac169c77ab944a03bfada70f00f7ac43", 1));
classifications.put("MediaType.Other", new WriteClassification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1));
classifications.put("MediaType.Banner", new WriteClassification("eb786339a5de44679ff6a7b4008f1ee0", 1));
@@ -91,6 +97,12 @@ public class Definitions {
classifications.put("MediaType.Video", new WriteClassification("4f9d38b889eb46f58d79a73100a2b55a", 1));
classifications.put("MediaType.Image", new WriteClassification("58f655cbd4214a72bb2aa73100a2a3dd", 1));
classifications.put("MediaType.Packshot", new WriteClassification("ba5a96a7d94245e48f14a73100a2a97c", 1));
classifications.put("MediaType.Guideline", new WriteClassification("a3c6732752274a99817ba7dd007a207f", 1));
classificationGuidToKeyMap = new HashMap<String, String>();
for (Entry<String, WriteClassification> entry : classifications.entrySet()) {
classificationGuidToKeyMap.put(entry.getValue().id, entry.getKey());
}
mediaTypes = new HashMap<Long, String>();
mediaTypes.put(1L, "MediaType.unassigned");
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+100 -40
View File
@@ -52,7 +52,7 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String adamHost = "https://edam-q.bayer.com:2015";
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 28861, 28860, 71948, /*71048, 70778, 70746, 70142,*/ 70361/*, 63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 28861, 28860, 71948, 71048, 70778, /*70746, 70142,*/ 70361/*, 63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
70288, 71725*/));
static final String categoryJson = "resources/2017-10-19_categories.json";
@@ -85,7 +85,7 @@ public class MamMigrationSelective implements CommandLineRunner {
@Override
public void run(String... strings) throws Exception {
// readEntryIDsFromFile();
readEntryIDsFromFile();
log.info("Write log headers for " + entryLog.getAbsolutePath());
FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
@@ -109,7 +109,7 @@ public class MamMigrationSelective implements CommandLineRunner {
for (Classification yearClassification : campaignYears) {
List<Classification> campaignClassifications = restTemplate.exchange(adamHost + "/classification/" + yearClassification.id + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
for (Classification campaignClassification : campaignClassifications) {
log.debug(yearClassification.name + ": " + campaignClassification.name + ", " + campaignClassification.labels[0].value + ", " + campaignClassification.id);
//log.debug(yearClassification.name + ": " + campaignClassification.name + ", " + campaignClassification.labels[0].value + ", " + campaignClassification.id);
Campaign campaign = new Campaign(campaignClassification.name, campaignClassification.labels[0].value, yearClassification.name);
campaign.guid = campaignClassification.id;
campaigns.add(campaign);
@@ -118,6 +118,18 @@ public class MamMigrationSelective implements CommandLineRunner {
HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
for (int id : entrySelection) {
// XXX
if (id > 80000 || id < 72048)
continue;
log.info("GET entry " + id);
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
@@ -125,16 +137,20 @@ public class MamMigrationSelective implements CommandLineRunner {
log.debug("Name: " + entry.name.get("en"));
// check exclusion criteria
if (entry.stateId == 7) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + "(LOCKED_BY_ADMIN)");
if ((entry.stateId == 5 || entry.stateId == 4) && entry.fileName == null && entry.categories.size() == 0) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + ", no file attached, no category set");
continue;
}
else if (entry.stateId == 7) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (LOCKED_BY_ADMIN)");
continue;
}
else if (entry.stateId == 8) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + "(ASSIGNED_TO_INDEXATION)");
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ASSIGNED_TO_INDEXATION)");
continue;
}
else if (entry.stateId == 10) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + "(ARCHIVED)");
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ARCHIVED)");
continue;
}
@@ -185,7 +201,39 @@ public class MamMigrationSelective implements CommandLineRunner {
}
if (n.action.equals("classify")) {
record.addClassification(n.parameter);
String[] parameters = n.parameter.split(",");
for (String p : parameters) {
// TODO what if "Released" is assigned after "Archived"? (Multiple category problem)
if (p.startsWith("Status.")) {
if (!p.equals("Status.Released"))
record.removeClassification("Status.Released");
else if (record.hasClassificationStartingWith("Status.")){
log.warn("Classification Status.Released not added, different status already set");
continue;
}
}
if (p.startsWith("BusinessUnit.")) {
record.removeClassification("BusinessUnit.CAP");
record.removeClassification("BusinessUnit.FAP");
record.removeClassification("BusinessUnit.General");
}
if (p.startsWith("Brand.")) {
// always use the most specific brand
if (p.equals("Brand.General") && record.hasClassificationStartingWith("Brand.")) {
log.warn("Classification Brand.General not added, different brand already set");
continue;
}
else {
record.removeClassification("Brand.General");
}
}
record.addClassification(p);
}
}
else if (n.action.equals("brand")) {
String brandClassificationId = "Brand." + n.name.trim();
@@ -273,7 +321,13 @@ public class MamMigrationSelective implements CommandLineRunner {
}
// Fields
record.addField("byr_Title", entry.name.get("en"));
if (entry.name.get("en") != null) {
record.addField("byr_Title", entry.name.get("en"));
}
else {
log.warn("Entry " + entry.id + ": No title");
record.addField("byr_Title", "No title");
}
// TODO
record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
@@ -289,8 +343,10 @@ public class MamMigrationSelective implements CommandLineRunner {
if (entry.comment.get("en") != null)
description += "Comment: " + entry.comment.get("en") + "\n\n";
String migrationDescription = "Migrated from Media Assistant ID " + entry.id + "\nDate of Issue: "
+ stringDate.format(entry.creationDate);
String migrationDescription = "Migrated from Media Assistant ID " + entry.id;
if (entry.creationDate != null)
migrationDescription += "\nDate of Issue: " + stringDate.format(entry.creationDate);
description += migrationDescription;
record.addField("byr_ah_description", description);
@@ -305,43 +361,47 @@ public class MamMigrationSelective implements CommandLineRunner {
// License Information
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
// if license holder is not "Bayer Animal Health GmbH" put on review list
if (!entry.copyright.organization.get("id").equals("3740")) {
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
}
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
String licenseDescription = "";
if (entry.copyright.use != null)
licenseDescription += entry.copyright.use;
record.addField("byr_ah_licenseComments", licenseDescription);
if (entry.copyright != null && entry.copyright.organization != null) {
String licenseDescription = "";
if (entry.copyright.use != null)
licenseDescription += entry.copyright.use;
record.addField("byr_ah_licenseComments", licenseDescription);
// if license holder is not "Bayer Animal Health GmbH" put on review list
if (!entry.copyright.organization.get("id").equals("3740"))
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
}
else {
// if license information is missing put on review list
log.warn("Entry " + entry.id + ": Copyright information missing");
}
// if (entry.gpc != null)
// record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber);
if (entry.gpc != null)
record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber);
// XXX
// Create record
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
}
catch (HttpStatusCodeException hsce) {
log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
log.error(hsce.getResponseBodyAsString());
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
}
// HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
// try {
// URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, UploadRecord.class);
// log.info("RESULT Entry " + entry.id + " migrated to record " + location);
//
// FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
// Charset.forName("utf-8"), true);
// }
// catch (HttpStatusCodeException hsce) {
// log.error("RESULT Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
// log.error(hsce.getResponseBodyAsString());
// FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
// }
}
private void readEntryIDsFromFile() throws IOException {
File idFile = new File("resources/entrylist_2017-10-25.txt");
log.debug("Reading entry IDs from " + idFile.getAbsolutePath());
List<String> entryIdList = FileUtils.readLines(idFile, Charset.forName("utf-8"));
File entryIdListFile = new File("resources/entrylist_2017-10-25.txt");
log.debug("Reading entry IDs from " + entryIdListFile.getAbsolutePath());
List<String> entryIdList = FileUtils.readLines(entryIdListFile, Charset.forName("utf-8"));
entrySelection = new ArrayList<Integer>();
AH MAM Migration/src/com/bayer/edam/migration/adam/UploadRecord.java
+17 -1
View File
@@ -86,6 +86,23 @@ public class UploadRecord {
}
}
public void removeClassification(String name) {
boolean removed = classifications.addOrUpdate.remove(Definitions.classifications.get(name));
if (removed)
log.debug("Removed classification " + name);
}
public boolean hasClassificationStartingWith(String prefix) {
for (WriteClassification c : classifications.addOrUpdate) {
String key = Definitions.classificationGuidToKeyMap.get(c.id);
if (key != null && key.startsWith(prefix))
return true;
}
return false;
}
public void addField(String name, String value) {
fields.addOrUpdate.add(Definitions.createSingleValueField(name, value));
}
@@ -93,5 +110,4 @@ public class UploadRecord {
public void addField(String name, List<String> values) {
fields.addOrUpdate.add(Definitions.createMultiValueField(name, values));
}
}