tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CategoryTree

Browse Source
This commit is contained in:
gexce
2017-09-28 19:09:11 +02:00
parent 22dc51df7a
commit 37ed0f291e
9 changed files with 236 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/questions.txt
+2 -1
View File
@@ -44,7 +44,7 @@
- Status: Liste mit Status != ACCEPTED an Claudia/Heike [done]
Archiv ist nur andere Kategorie
- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin/Tilman klären Prozess
- Seite 2 als Preview
- Seite 2 als Preview --> möglich?
- Filename Preview nach Feld fileName setzen
-----------------------------------------------------
Status „Archived“ kann gelöscht werden bzw. muss nicht übernommen werden.
@@ -56,6 +56,7 @@ Status „Zum Upload zugewiesen“ bitte behalten.
CREATE TABLE Record (
ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY,
Location VARCHAR NOT NULL,
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -1,4 +1,4 @@
# TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
logging.level.com.bayer=DEBUG
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=entryDownload.log
logging.file=entryDownload.log
AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java
+7 -33
View File
@@ -38,6 +38,13 @@ public class EntryProcessor {
while (iter.hasNext()) {
String line = iter.next();
String status = line.substring(19, line.indexOf(fieldSeparator, 20));
if (status.equals("10") || status.equals("7") || status.equals("8")) {
count++;
continue;
}
line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
Entry entry = jsonMapper.readValue(line, Entry.class);
@@ -88,39 +95,6 @@ public class EntryProcessor {
aggregatedFilesize += download.size;
}
/*
HashSet<String> filenames = new HashSet<String>();
HashSet<String> hrefs = new HashSet<String>();
for (Download download : downloadSelection) {
String filenameToUse;
if (download.fileNameOnDisc != null)
filenameToUse = download.fileNameOnDisc;
else
filenameToUse = download.fileName;
if (filenameToUse == null) {
log.error("ARGH " + entry.id);
System.exit(1);
}
if (filenames.add(filenameToUse)) {
aggregatedFilesize += download.size;
}
hrefs.add(download.href);
}
if (filenames.size() < hrefs.size()) {
log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size());
for (String s : filenames)
log.error(s);
for (String s : hrefs)
log.error(s);
}
*/
}
log.info("Count: " + count);
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+51 -25
View File
@@ -42,7 +42,7 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String adamHost = "https://edam-q.bayer.com:2015";
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
static final int[] selection = { /*71948, 71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
70288, 71725*/ };
static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log");
@@ -87,8 +87,22 @@ public class MamMigrationSelective implements CommandLineRunner {
log.info("GET entry " + id);
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
// postToAdam(entry);
// check exclusion criteria
if (entry.stateId == 7) {
log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(LOCKED_BY_ADMIN)");
continue;
}
else if (entry.stateId == 8) {
log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(ASSIGNED_TO_INDEXATION)");
continue;
}
else if (entry.stateId == 10) {
log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(ARCHIVED)");
continue;
}
postToAdam(entry);
}
// // delete all created records
@@ -110,24 +124,32 @@ public class MamMigrationSelective implements CommandLineRunner {
record.addClassification("Country.Global");
record.addClassification("Agencies.Bayer AH");
record.addClassification("Usage.Restricted");
// status
if (entry.stateId == 6) { // ACCEPTED
record.addClassification("Status.Released");
} else if (entry.stateId == 10) { // ARCHIVED
record.addClassification("Status.Archived");
} else {
log.error("Skipping entry " + entry.id + ": No target status for stateId == " + entry.stateId);
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
return;
}
// business unit
record.addClassification("BusinessUnit.CAP");
// media type
record.addClassification(Definitions.mediaTypes.get(entry.mediaType.id));
// process table
Set<String> keywords = new HashSet<String>();
String status = "Status.Released";
String businessUnit = "BusinessUnit.General";
for (int category : entry.categories) {
// TODO
}
record.addClassification(status);
record.addClassification(businessUnit);
// Fields
record.addField("byr_Title", entry.name.get("en"));
@@ -152,19 +174,21 @@ public class MamMigrationSelective implements CommandLineRunner {
// Keywords
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
Set<String> keywords = new HashSet<String>(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
keywords.addAll(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
}
if (keywords.size() > 0) {
record.addField("byr_ah_keywords_tl", new ArrayList<String>(keywords));
}
// License Information
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
// SR: If license holder is not "Bayer Animal Health GmbH" put on review list
// if license holder is not "Bayer Animal Health GmbH" put on review list
if (!entry.copyright.organization.get("id").equals("3740")) {
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
}
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
String licenseDescription = "";
@@ -172,9 +196,12 @@ public class MamMigrationSelective implements CommandLineRunner {
licenseDescription += entry.copyright.use;
record.addField("byr_ah_licenseComments", licenseDescription);
if (entry.gpc != null)
record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); // TODO
// if (entry.gpc != null)
// record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber);
// Create record
HttpEntity<WriteRecord> httpEntity = new HttpEntity<WriteRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class);
@@ -209,8 +236,7 @@ public class MamMigrationSelective implements CommandLineRunner {
headers.add("Accept", "application/hal+json");
headers.add("Content-Type", "application/json");
headers.add("Authorization", "Basic " + adamAuth);
headers.add("set-immediateSearchIndexUpdate", "true"); // see
// https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
HttpEntity<String> entity = new HttpEntity<String>("", headers);
Token token = restTemplate.exchange(adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody();
AH MAM Migration/src/com/bayer/edam/migration/attic/Tesssst.java
+62
View File
@@ -0,0 +1,62 @@
package com.bayer.edam.migration.attic;
import java.util.ArrayList;
import java.util.List;
public class Tesssst {
public static void main(String[] args) {
List<String> entryCategories = new ArrayList<String>();
entryCategories.add("aaa/bbb");
entryCategories.add("aaa/bbb/ccc/ddd");
entryCategories.add("aaa");
entryCategories.add("xxx/yyy/zzz");
entryCategories.add("xxx/yyy/zzz/nnn");
entryCategories.add("xxx/yyy/zzz");
entryCategories.add("xxx/yyy/zzz");
entryCategories.add("xxx/yyy/zzz/ttt");
List<String> categories = new ArrayList<String>();
for (String entryCategory : entryCategories) {
List<String> tmpCategories = new ArrayList<String>();
boolean match = false;
for (String category : categories) {
if (match) {
System.out.println("Adding " + category);
tmpCategories.add(category);
continue;
}
if (category.length() < entryCategory.length() && category.equals(entryCategory.substring(0, category.length()))) {
System.out.println("Adding " + entryCategory);
tmpCategories.add(entryCategory);
match = true;
}
else if (category.length() >= entryCategory.length() && entryCategory.equals(category.substring(0, entryCategory.length()))) {
System.out.println("Adding " + category);
tmpCategories.add(category);
match = true;
}
else {
System.out.println("Adding " + category);
tmpCategories.add(category);
}
}
if (!match) {
System.out.println("Adding " + entryCategory);
tmpCategories.add(entryCategory);
match = true;
}
categories = tmpCategories;
}
System.out.println("\nOutcome:");
for (String s : categories) {
System.out.println(s);
}
}
}
AH MAM Migration/src/com/bayer/edam/migration/mam/Category.java
-14
View File
@@ -1,14 +0,0 @@
package com.bayer.edam.migration.mam;
import java.util.Map;
public class Category {
public Long id;
public Integer parent;
public Map<String, String> name;
public Category() {
}
}
AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java
+1 -1
View File
@@ -86,7 +86,7 @@ public class Entry {
public Gpc gpc;
public String mediaAssistantId;
public Map<String, String> metaData;
public List<String> categories;
public List<Integer> categories;
public Integer stateId;
public Integer contentSuperTypeId;
public List<Download> downloadList;
AH MAM Migration/src/com/bayer/edam/migration/mam/EntrySimple.java
-34
View File
@@ -1,34 +0,0 @@
package com.bayer.edam.migration.mam;
import java.util.Date;
import java.util.Map;
public class EntrySimple {
public Long id;
public String href;
public Long fileSize;
public String fileName;
public Date creationDate;
public Date uploadDate;
public Map<String, String> name;
public Map<String, String> keyword;
public Map<String, String> description;
public Map<String, String> comment;
public Map<String, String> user;
public Map<String, String> uploader;
public Map<String, String> indexer;
// public Double ratingValue;
// public Map<String, String> release;
// public String brandingNet;
public EntrySimple() {
}
@Override
public String toString() {
return "Entry{" + "id=" + id + ", href=" + href + '}';
}
}
AH MAM Migration/src/com/bayer/edam/migration/mam/categories/CategoryTree.java
+112
View File
@@ -0,0 +1,112 @@
package com.bayer.edam.migration.mam.categories;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.io.FileUtils;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
public class CategoryTree {
static class Node implements Comparable<Node> {
int id;
String name;
Node parent;
int parentId;
List<Node> children = new ArrayList<Node>();
@Override
public int compareTo(Node o) {
return this.name.compareTo(o.name);
}
}
static class JsonCategory {
public int id;
public int parent;
public Map<String, String> name;
public JsonCategory() {}
}
static class JsonCategoryList {
public List<JsonCategory> nodes;
public JsonCategoryList() {}
}
Node root;
public CategoryTree(String jsonFile) throws IOException {
ObjectMapper jsonMapper = new ObjectMapper();
// String categoriesString = FileUtils.readFileToString(new File(jsonFile), "UTF-8");
// JsonCategoryList categoryList = jsonMapper.readValue(categoriesString, JsonCategoryList.class);
JavaType type = jsonMapper.getTypeFactory().constructCollectionType(List.class, JsonCategory.class);
List<JsonCategory> categoryList = jsonMapper.readValue(new File(jsonFile), type);
HashMap<Integer, Node> nodeMap = new HashMap<Integer, Node>();
for (JsonCategory cat : categoryList) {
Node node = new Node();
node.id = cat.id;
node.parentId = cat.parent;
node.name = cat.name.get("en");
nodeMap.put(node.id, node);
if (cat.id == 0) {
root = node;
}
}
for (Entry<Integer,Node> entry : nodeMap.entrySet()) {
Node node = entry.getValue();
node.parent = nodeMap.get(node.parentId);
if (node != node.parent) {
node.parent.children.add(node);
}
}
// printout
System.out.println("Categories:");
Node startNode = nodeMap.get(1);
java.util.Collections.sort(startNode.children);
for (Node node : startNode.children) {
iterate(node, 0);
}
}
private void iterate(Node node, int depth) {
for (int i = 0; i < depth; i++) {
System.out.print(";");
}
System.out.println(node.name);
if (node.children.size() > 0) {
java.util.Collections.sort(node.children);
for (Node child : node.children) {
iterate(child, depth+1);
}
}
}
public static void main(String[] args) {
try {
new CategoryTree("C:\\Temp\\Migration AH\\categories.json");
} catch (IOException e) {
e.printStackTrace();
}
}
}