From 5209e725e2c93a1b1b4b61a94fc13519e3f29899 Mon Sep 17 00:00:00 2001 From: gexce Date: Mon, 18 Dec 2017 11:23:21 +0100 Subject: [PATCH] EntryListAnalyzer --- .../.settings/org.eclipse.jdt.core.prefs | 11 --- .../resources/application.properties | 2 +- .../edam/migration/EntryListAnalyzer.java | 92 +++++++++++++++++++ .../edam/migration/MamMigrationSelective.java | 39 ++++---- .../edam/migration/mam/CategoryTree.java | 2 +- 5 files changed, 116 insertions(+), 30 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/EntryListAnalyzer.java diff --git a/AH MAM Migration/.settings/org.eclipse.jdt.core.prefs b/AH MAM Migration/.settings/org.eclipse.jdt.core.prefs index 529ef07..fd9afef 100644 --- a/AH MAM Migration/.settings/org.eclipse.jdt.core.prefs +++ b/AH MAM Migration/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,2 @@ eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.8 diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index ef69f62..94f3fc0 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -2,4 +2,4 @@ logging.level.com.bayer=DEBUG #logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application_2017-11-28_PROD_fix.log +logging.file=application_2017-12-18_Nach-Import.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryListAnalyzer.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryListAnalyzer.java new file mode 100644 index 0000000..9bc28c9 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryListAnalyzer.java @@ -0,0 +1,92 @@ +package com.bayer.edam.migration; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import com.bayer.edam.migration.mam.CategoryTree; +import com.bayer.edam.migration.mam.CategoryTree.Node; +import com.bayer.edam.migration.mam.Entry; + +//@SpringBootApplication +public class EntryListAnalyzer implements CommandLineRunner { + + private static final Logger log = LoggerFactory.getLogger(EntryListAnalyzer.class); + + static final String entrySelectionList = "C:\\Temp\\Migration AH\\Migration-Logs PROD\\Logs and Analysis\\2017-12-14 not migrated entires.txt"; + static final String entryList = "resources/entrylist_FULL_2017-11-15.txt"; + static final String categoryJson = "resources/2017-10-19_categories.json"; + static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv"; + + + private HashMap entryMap; + private CategoryTree mamCategoryTree; + + static final String fieldSeparator = "|~|"; + + @Override + public void run(String... args) throws Exception { + entryMap = MamMigrationSelective.readEntriesFromFile(entryList); + mamCategoryTree = new CategoryTree(categoryJson, categoryMapping); + + List entrySelection = new ArrayList(); + List entryIDsList = FileUtils.readLines(new File(entrySelectionList), Charset.forName("utf-8")); + for (String idString : entryIDsList) { + entrySelection.add(Long.parseLong(idString)); + } + + for (long id : entrySelection) { + Entry entry = entryMap.get(id); + + if (entry == null) { + log.error("Could not find entry for ID " + id); + continue; + } + + if (entry.categories.size() > 1) { + String categoriesList = ""; + for (int i = 0; i < entry.categories.size(); i++) { + + Node node = mamCategoryTree.getNode(entry.categories.get(i)); + + Node tmpNode = node; + if (tmpNode.id == CategoryTree.ROOT_NODE_ID) { + continue; + } + + boolean skipCategory = false; + while (tmpNode.id != CategoryTree.ROOT_NODE_ID) { + if (tmpNode.id == 0 || tmpNode.action.equals("not migrated")) { + skipCategory = true; + } + tmpNode = tmpNode.parent; + } + + if (!skipCategory) { + if (categoriesList.length() > 0) + categoriesList += ", "; + categoriesList += node.name; + } + } + + if (categoriesList.length() > 0) + log.info(entry.id + " : " + categoriesList); + } + } + + } + + public static void main(String args[]) { + SpringApplication.run(EntryListAnalyzer.class, args); + } + +} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 99411ae..4b63238 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -104,7 +104,7 @@ public class MamMigrationSelective implements CommandLineRunner { private static DecimalFormat decimalFormat = new DecimalFormat("#,##0.#"); private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); - private ObjectMapper jsonMapper = new ObjectMapper(); + static ObjectMapper jsonMapper = new ObjectMapper(); private HashMap entryMap; private List campaignYears; @@ -129,15 +129,15 @@ public class MamMigrationSelective implements CommandLineRunner { // Option 1: Process all available entries // entrySelection = new ArrayList(entryMap.keySet()); - // Option 2: Process a fixed set of entry IDs - entrySelection = Arrays.asList(55803L, 55802L); +// // Option 2: Process a fixed set of entry IDs +// entrySelection = Arrays.asList(55803L, 55802L); // Option 3: Read entry IDs from file -// entrySelection = new ArrayList(); -// List entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8")); -// for (String idString : entryIDsList) { -// entrySelection.add(Long.parseLong(idString)); -// } + entrySelection = new ArrayList(); + List entryIDsList = FileUtils.readLines(new File("resources/2017-12-15 Nach-Import - Entries die wegen einer von mehreren Kategorien ausgeschlossen wurden.txt"), Charset.forName("utf-8")); + for (String idString : entryIDsList) { + entrySelection.add(Long.parseLong(idString)); + } // -------------------------------------------------------------------- Collections.sort(entrySelection); @@ -174,11 +174,11 @@ public class MamMigrationSelective implements CommandLineRunner { // XXX - // if needed start from a certain ID - if (id > 58002 || id < 35620) { - skippedCounter++; - continue; - } +// // if needed start from a certain ID +// if (id > 58002 || id < 35620) { +// skippedCounter++; +// continue; +// } // refresh login every 4 hours if (System.currentTimeMillis() - lastLogin > 14400000) @@ -257,6 +257,7 @@ public class MamMigrationSelective implements CommandLineRunner { if (entry.categories.size() > 1) log.warn("Entry " + entry.id + " has " + entry.categories.size() + " categories assigned"); + categories: for (int categoryId : entry.categories) { log.debug("Category: " + categoryId); @@ -285,9 +286,13 @@ public class MamMigrationSelective implements CommandLineRunner { for (Node n : path) { if (n.action.equals("not migrated")) { - log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")"); - skippedCounter++; - return; +// log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")"); +// skippedCounter++; +// return; + + // XXX Nach-Import + log.info("Removing category " + categoryId + ", because of node " + n); + continue categories; } if (n.action.equals("classify")) { @@ -919,7 +924,7 @@ public class MamMigrationSelective implements CommandLineRunner { return file; } - private HashMap readEntriesFromFile(String filePath) throws IOException { + protected static HashMap readEntriesFromFile(String filePath) throws IOException { File entryListFile = new File(filePath); log.debug("Reading entries from " + entryListFile.getAbsolutePath()); List entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8")); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/CategoryTree.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/CategoryTree.java index 6a3523b..2701fec 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/CategoryTree.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/CategoryTree.java @@ -101,7 +101,7 @@ public class CategoryTree { rowNum++; if (row[0] == null || row[0].equals("Action")) { - log.info(rowNum + " - skipping, first cell ist empty or heading"); + log.info(rowNum + " - skipping, first cell is empty or heading"); continue; }