From 37ed0f291e1650d58eda0b7b5be6fd70a3f5ef96 Mon Sep 17 00:00:00 2001 From: gexce Date: Thu, 28 Sep 2017 19:09:11 +0200 Subject: [PATCH] CategoryTree --- AH MAM Migration/questions.txt | 3 +- .../resources/application.properties | 2 +- .../bayer/edam/migration/EntryProcessor.java | 40 ++----- .../edam/migration/MamMigrationSelective.java | 76 ++++++++---- .../bayer/edam/migration/attic/Tesssst.java | 62 ++++++++++ .../bayer/edam/migration/mam/Category.java | 14 --- .../com/bayer/edam/migration/mam/Entry.java | 2 +- .../bayer/edam/migration/mam/EntrySimple.java | 34 ------ .../mam/categories/CategoryTree.java | 112 ++++++++++++++++++ 9 files changed, 236 insertions(+), 109 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/attic/Tesssst.java delete mode 100644 AH MAM Migration/src/com/bayer/edam/migration/mam/Category.java delete mode 100644 AH MAM Migration/src/com/bayer/edam/migration/mam/EntrySimple.java create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/mam/categories/CategoryTree.java diff --git a/AH MAM Migration/questions.txt b/AH MAM Migration/questions.txt index c3ac221..78823a1 100644 --- a/AH MAM Migration/questions.txt +++ b/AH MAM Migration/questions.txt @@ -44,7 +44,7 @@ - Status: Liste mit Status != ACCEPTED an Claudia/Heike [done] Archiv ist nur andere Kategorie - Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin/Tilman klären Prozess -- Seite 2 als Preview +- Seite 2 als Preview --> möglich? - Filename Preview nach Feld fileName setzen ----------------------------------------------------- Status „Archived“ kann gelöscht werden bzw. muss nicht übernommen werden. @@ -56,6 +56,7 @@ Status „Zum Upload zugewiesen“ bitte behalten. + CREATE TABLE Record ( ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, Location VARCHAR NOT NULL, diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index b1ed2f0..e71cd36 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -1,4 +1,4 @@ # TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF logging.level.com.bayer=DEBUG logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=entryDownload.log \ No newline at end of file +logging.file=entryDownload.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java index 4405fd7..8309737 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java @@ -38,6 +38,13 @@ public class EntryProcessor { while (iter.hasNext()) { String line = iter.next(); + + String status = line.substring(19, line.indexOf(fieldSeparator, 20)); + if (status.equals("10") || status.equals("7") || status.equals("8")) { + count++; + continue; + } + line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length()); Entry entry = jsonMapper.readValue(line, Entry.class); @@ -88,39 +95,6 @@ public class EntryProcessor { aggregatedFilesize += download.size; } - - - - /* - HashSet filenames = new HashSet(); - HashSet hrefs = new HashSet(); - for (Download download : downloadSelection) { - String filenameToUse; - if (download.fileNameOnDisc != null) - filenameToUse = download.fileNameOnDisc; - else - filenameToUse = download.fileName; - - if (filenameToUse == null) { - log.error("ARGH " + entry.id); - System.exit(1); - } - - if (filenames.add(filenameToUse)) { - aggregatedFilesize += download.size; - } - hrefs.add(download.href); - } - - if (filenames.size() < hrefs.size()) { - log.error("Problem with " + entry.id + " filenames.size=" + filenames.size() + ", hrefs.size=" + hrefs.size()); - for (String s : filenames) - log.error(s); - for (String s : hrefs) - log.error(s); - } - */ - } log.info("Count: " + count); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 8e71095..ad9109c 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -42,7 +42,7 @@ public class MamMigrationSelective implements CommandLineRunner { static final String adamHost = "https://edam-q.bayer.com:2015"; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; - static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340, + static final int[] selection = { /*71948, 71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340, 70288, 71725*/ }; static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log"); @@ -87,8 +87,22 @@ public class MamMigrationSelective implements CommandLineRunner { log.info("GET entry " + id); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); - -// postToAdam(entry); + + // check exclusion criteria + if (entry.stateId == 7) { + log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(LOCKED_BY_ADMIN)"); + continue; + } + else if (entry.stateId == 8) { + log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(ASSIGNED_TO_INDEXATION)"); + continue; + } + else if (entry.stateId == 10) { + log.info("Skipping entry " + entry.id + ": Status " + entry.stateId + "(ARCHIVED)"); + continue; + } + + postToAdam(entry); } // // delete all created records @@ -110,24 +124,32 @@ public class MamMigrationSelective implements CommandLineRunner { record.addClassification("Country.Global"); record.addClassification("Agencies.Bayer AH"); record.addClassification("Usage.Restricted"); - - // status - if (entry.stateId == 6) { // ACCEPTED - record.addClassification("Status.Released"); - } else if (entry.stateId == 10) { // ARCHIVED - record.addClassification("Status.Archived"); - } else { - log.error("Skipping entry " + entry.id + ": No target status for stateId == " + entry.stateId); - FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true); - return; - } - - // business unit - record.addClassification("BusinessUnit.CAP"); - + // media type record.addClassification(Definitions.mediaTypes.get(entry.mediaType.id)); + + + + // process table + Set keywords = new HashSet(); + String status = "Status.Released"; + String businessUnit = "BusinessUnit.General"; + + for (int category : entry.categories) { + + // TODO + + + + } + + record.addClassification(status); + record.addClassification(businessUnit); + + + + // Fields record.addField("byr_Title", entry.name.get("en")); @@ -152,19 +174,21 @@ public class MamMigrationSelective implements CommandLineRunner { // Keywords if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) { - Set keywords = new HashSet(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"))); + keywords.addAll(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"))); + } + + if (keywords.size() > 0) { record.addField("byr_ah_keywords_tl", new ArrayList(keywords)); } // License Information record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder)); - // SR: If license holder is not "Bayer Animal Health GmbH" put on review list + // if license holder is not "Bayer Animal Health GmbH" put on review list if (!entry.copyright.organization.get("id").equals("3740")) { log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)"); } - record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse)); String licenseDescription = ""; @@ -172,9 +196,12 @@ public class MamMigrationSelective implements CommandLineRunner { licenseDescription += entry.copyright.use; record.addField("byr_ah_licenseComments", licenseDescription); - if (entry.gpc != null) - record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); // TODO +// if (entry.gpc != null) +// record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); + + + // Create record HttpEntity httpEntity = new HttpEntity(record, adamHeaders); try { URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class); @@ -209,8 +236,7 @@ public class MamMigrationSelective implements CommandLineRunner { headers.add("Accept", "application/hal+json"); headers.add("Content-Type", "application/json"); headers.add("Authorization", "Basic " + adamAuth); - headers.add("set-immediateSearchIndexUpdate", "true"); // see - // https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push + headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push HttpEntity entity = new HttpEntity("", headers); Token token = restTemplate.exchange(adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody(); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/attic/Tesssst.java b/AH MAM Migration/src/com/bayer/edam/migration/attic/Tesssst.java new file mode 100644 index 0000000..49ccc1a --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/attic/Tesssst.java @@ -0,0 +1,62 @@ +package com.bayer.edam.migration.attic; + +import java.util.ArrayList; +import java.util.List; + +public class Tesssst { + + public static void main(String[] args) { + List entryCategories = new ArrayList(); + entryCategories.add("aaa/bbb"); + entryCategories.add("aaa/bbb/ccc/ddd"); + entryCategories.add("aaa"); + entryCategories.add("xxx/yyy/zzz"); + entryCategories.add("xxx/yyy/zzz/nnn"); + entryCategories.add("xxx/yyy/zzz"); + entryCategories.add("xxx/yyy/zzz"); + entryCategories.add("xxx/yyy/zzz/ttt"); + + List categories = new ArrayList(); + for (String entryCategory : entryCategories) { + List tmpCategories = new ArrayList(); + boolean match = false; + for (String category : categories) { + if (match) { + System.out.println("Adding " + category); + tmpCategories.add(category); + continue; + } + + if (category.length() < entryCategory.length() && category.equals(entryCategory.substring(0, category.length()))) { + System.out.println("Adding " + entryCategory); + tmpCategories.add(entryCategory); + match = true; + } + else if (category.length() >= entryCategory.length() && entryCategory.equals(category.substring(0, entryCategory.length()))) { + System.out.println("Adding " + category); + tmpCategories.add(category); + match = true; + } + else { + System.out.println("Adding " + category); + tmpCategories.add(category); + } + } + + if (!match) { + System.out.println("Adding " + entryCategory); + tmpCategories.add(entryCategory); + match = true; + } + + categories = tmpCategories; + } + + System.out.println("\nOutcome:"); + for (String s : categories) { + System.out.println(s); + } + + } + +} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/Category.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/Category.java deleted file mode 100644 index 0ef7516..0000000 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/Category.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.bayer.edam.migration.mam; - -import java.util.Map; - -public class Category { - - public Long id; - public Integer parent; - public Map name; - - public Category() { - } - -} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java index 753b816..4d1d619 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java @@ -86,7 +86,7 @@ public class Entry { public Gpc gpc; public String mediaAssistantId; public Map metaData; - public List categories; + public List categories; public Integer stateId; public Integer contentSuperTypeId; public List downloadList; diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/EntrySimple.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/EntrySimple.java deleted file mode 100644 index 1a9ec70..0000000 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/EntrySimple.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.bayer.edam.migration.mam; - -import java.util.Date; -import java.util.Map; - -public class EntrySimple { - - public Long id; - public String href; - public Long fileSize; - public String fileName; - public Date creationDate; - public Date uploadDate; - public Map name; - public Map keyword; - public Map description; - public Map comment; - public Map user; - public Map uploader; - public Map indexer; -// public Double ratingValue; -// public Map release; -// public String brandingNet; - - public EntrySimple() { - } - - @Override - public String toString() { - return "Entry{" + "id=" + id + ", href=" + href + '}'; - } - - -} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/categories/CategoryTree.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/categories/CategoryTree.java new file mode 100644 index 0000000..30f389b --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/categories/CategoryTree.java @@ -0,0 +1,112 @@ +package com.bayer.edam.migration.mam.categories; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.io.FileUtils; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class CategoryTree { + + static class Node implements Comparable { + int id; + String name; + Node parent; + int parentId; + List children = new ArrayList(); + + @Override + public int compareTo(Node o) { + return this.name.compareTo(o.name); + } + } + + static class JsonCategory { + public int id; + public int parent; + public Map name; + + public JsonCategory() {} + } + + static class JsonCategoryList { + public List nodes; + + public JsonCategoryList() {} + } + + + Node root; + + public CategoryTree(String jsonFile) throws IOException { + ObjectMapper jsonMapper = new ObjectMapper(); + +// String categoriesString = FileUtils.readFileToString(new File(jsonFile), "UTF-8"); +// JsonCategoryList categoryList = jsonMapper.readValue(categoriesString, JsonCategoryList.class); + + JavaType type = jsonMapper.getTypeFactory().constructCollectionType(List.class, JsonCategory.class); + List categoryList = jsonMapper.readValue(new File(jsonFile), type); + + HashMap nodeMap = new HashMap(); + + for (JsonCategory cat : categoryList) { + Node node = new Node(); + node.id = cat.id; + node.parentId = cat.parent; + node.name = cat.name.get("en"); + nodeMap.put(node.id, node); + + if (cat.id == 0) { + root = node; + } + } + + for (Entry entry : nodeMap.entrySet()) { + Node node = entry.getValue(); + node.parent = nodeMap.get(node.parentId); + + if (node != node.parent) { + node.parent.children.add(node); + } + } + + // printout + System.out.println("Categories:"); + Node startNode = nodeMap.get(1); + java.util.Collections.sort(startNode.children); + for (Node node : startNode.children) { + iterate(node, 0); + } + + } + + private void iterate(Node node, int depth) { + for (int i = 0; i < depth; i++) { + System.out.print(";"); + } + System.out.println(node.name); + + if (node.children.size() > 0) { + java.util.Collections.sort(node.children); + for (Node child : node.children) { + iterate(child, depth+1); + } + } + } + + public static void main(String[] args) { + try { + new CategoryTree("C:\\Temp\\Migration AH\\categories.json"); + } catch (IOException e) { + e.printStackTrace(); + } + } + +}