tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EntryListAnalyzer

Browse Source
This commit is contained in:
gexce
2017-12-18 11:23:21 +01:00
parent 88ad66867b
commit 5209e725e2
5 changed files with 116 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/.settings/org.eclipse.jdt.core.prefs
-11
View File
@@ -1,13 +1,2 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application_2017-11-28_PROD_fix.log
logging.file=application_2017-12-18_Nach-Import.log
AH MAM Migration/src/com/bayer/edam/migration/EntryListAnalyzer.java
+92
View File
@@ -0,0 +1,92 @@
package com.bayer.edam.migration;
import java.io.File;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import com.bayer.edam.migration.mam.CategoryTree;
import com.bayer.edam.migration.mam.CategoryTree.Node;
import com.bayer.edam.migration.mam.Entry;
//@SpringBootApplication
public class EntryListAnalyzer implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(EntryListAnalyzer.class);
static final String entrySelectionList = "C:\\Temp\\Migration AH\\Migration-Logs PROD\\Logs and Analysis\\2017-12-14 not migrated entires.txt";
static final String entryList = "resources/entrylist_FULL_2017-11-15.txt";
static final String categoryJson = "resources/2017-10-19_categories.json";
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
private HashMap<Long, Entry> entryMap;
private CategoryTree mamCategoryTree;
static final String fieldSeparator = "|~|";
@Override
public void run(String... args) throws Exception {
entryMap = MamMigrationSelective.readEntriesFromFile(entryList);
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
List<Long> entrySelection = new ArrayList<Long>();
List<String> entryIDsList = FileUtils.readLines(new File(entrySelectionList), Charset.forName("utf-8"));
for (String idString : entryIDsList) {
entrySelection.add(Long.parseLong(idString));
}
for (long id : entrySelection) {
Entry entry = entryMap.get(id);
if (entry == null) {
log.error("Could not find entry for ID " + id);
continue;
}
if (entry.categories.size() > 1) {
String categoriesList = "";
for (int i = 0; i < entry.categories.size(); i++) {
Node node = mamCategoryTree.getNode(entry.categories.get(i));
Node tmpNode = node;
if (tmpNode.id == CategoryTree.ROOT_NODE_ID) {
continue;
}
boolean skipCategory = false;
while (tmpNode.id != CategoryTree.ROOT_NODE_ID) {
if (tmpNode.id == 0 || tmpNode.action.equals("not migrated")) {
skipCategory = true;
}
tmpNode = tmpNode.parent;
}
if (!skipCategory) {
if (categoriesList.length() > 0)
categoriesList += ", ";
categoriesList += node.name;
}
}
if (categoriesList.length() > 0)
log.info(entry.id + " : " + categoriesList);
}
}
}
public static void main(String args[]) {
SpringApplication.run(EntryListAnalyzer.class, args);
}
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+22 -17
View File
@@ -104,7 +104,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private static DecimalFormat decimalFormat = new DecimalFormat("#,##0.#");
private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private ObjectMapper jsonMapper = new ObjectMapper();
static ObjectMapper jsonMapper = new ObjectMapper();
private HashMap<Long, Entry> entryMap;
private List<Classification> campaignYears;
@@ -129,15 +129,15 @@ public class MamMigrationSelective implements CommandLineRunner {
// Option 1: Process all available entries
// entrySelection = new ArrayList<Long>(entryMap.keySet());
// Option 2: Process a fixed set of entry IDs
entrySelection = Arrays.asList(55803L, 55802L);
// // Option 2: Process a fixed set of entry IDs
// entrySelection = Arrays.asList(55803L, 55802L);
// Option 3: Read entry IDs from file
// entrySelection = new ArrayList<Long>();
// List<String> entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8"));
// for (String idString : entryIDsList) {
// entrySelection.add(Long.parseLong(idString));
// }
entrySelection = new ArrayList<Long>();
List<String> entryIDsList = FileUtils.readLines(new File("resources/2017-12-15 Nach-Import - Entries die wegen einer von mehreren Kategorien ausgeschlossen wurden.txt"), Charset.forName("utf-8"));
for (String idString : entryIDsList) {
entrySelection.add(Long.parseLong(idString));
}
// --------------------------------------------------------------------
Collections.sort(entrySelection);
@@ -174,11 +174,11 @@ public class MamMigrationSelective implements CommandLineRunner {
// XXX
// if needed start from a certain ID
if (id > 58002 || id < 35620) {
skippedCounter++;
continue;
}
// // if needed start from a certain ID
// if (id > 58002 || id < 35620) {
// skippedCounter++;
// continue;
// }
// refresh login every 4 hours
if (System.currentTimeMillis() - lastLogin > 14400000)
@@ -257,6 +257,7 @@ public class MamMigrationSelective implements CommandLineRunner {
if (entry.categories.size() > 1)
log.warn("Entry " + entry.id + " has " + entry.categories.size() + " categories assigned");
categories:
for (int categoryId : entry.categories) {
log.debug("Category: " + categoryId);
@@ -285,9 +286,13 @@ public class MamMigrationSelective implements CommandLineRunner {
for (Node n : path) {
if (n.action.equals("not migrated")) {
log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")");
skippedCounter++;
return;
// log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")");
// skippedCounter++;
// return;
// XXX Nach-Import
log.info("Removing category " + categoryId + ", because of node " + n);
continue categories;
}
if (n.action.equals("classify")) {
@@ -919,7 +924,7 @@ public class MamMigrationSelective implements CommandLineRunner {
return file;
}
private HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
protected static HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
File entryListFile = new File(filePath);
log.debug("Reading entries from " + entryListFile.getAbsolutePath());
List<String> entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8"));
AH MAM Migration/src/com/bayer/edam/migration/mam/CategoryTree.java
+1 -1
View File
@@ -101,7 +101,7 @@ public class CategoryTree {
rowNum++;
if (row[0] == null || row[0].equals("Action")) {
log.info(rowNum + " - skipping, first cell ist empty or heading");
log.info(rowNum + " - skipping, first cell is empty or heading");
continue;
}