From 59697c22d7ca23e31d9b4a48675d91b54d31f11a Mon Sep 17 00:00:00 2001 From: gexce Date: Thu, 21 Sep 2017 15:02:59 +0200 Subject: [PATCH] EntryProcessor --- AH MAM Migration/questions.txt | 42 +++++++++--- .../com/bayer/edam/migration/Definitions.java | 5 +- .../bayer/edam/migration/EntryDownloader.java | 2 +- .../bayer/edam/migration/EntryProcessor.java | 65 +++++++++++++++++++ .../edam/migration/MamMigrationSelective.java | 16 +++-- .../edam/migration/attic/LogFileCleaner.java | 44 +++++++++++++ 6 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java diff --git a/AH MAM Migration/questions.txt b/AH MAM Migration/questions.txt index 366c9d1..72c6a7a 100644 --- a/AH MAM Migration/questions.txt +++ b/AH MAM Migration/questions.txt @@ -1,30 +1,54 @@ +1. Status +- Übertragen der Datensätze funktioniert grundsätzlich +- MAM -> JSON (komme darauf zurück) -> eDAM +- Felder da, suchbar +- Es fehlen noch die verlinkten Strukturen: Kategorien, Benutzer + +2. Fragen - English only? - CreationDate ist read-only - Mapping Media Type --> bringt Martin mit - Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder? -- Soll muss der MAM-Uploader irgendwo gespeichert werden? +- Soll muss der MAM-Uploader irgendwo gespeichert werden? --> Additional Info - "Bayer Animal Health GmbH" immer als License Holder OK? - "comment" mit in description? - MAM-Status "ARCHIVED" --> eDAM.Archived? - Was ist mit Status != ACCEPTED or ARCHIVED? - Regelung Cutover: Kein Upload mehr ab KW42 (16.10.) -License: + License: - Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763) --> mandatory - release.forInternet/forIntranet --> License comments - GPC owner/e-mail? -GPC: + GPC: - Beispiel für ausgefüllte GPC-Daten/Wird GPC-Block verwendet? - was ist mit regionalLicenseComment/personalLicenseComment? - Ist "Date of Issue" Release Date? Ist gpc.expirationDate = LMR Expiration Date? - GPC No. = LMR ID? +3. Timeline +- Fertigstellung Migrations-Programm +- Download/Upload Metadaten +- Download/Upload Dateien +- Download = knapp 4 Tage, Upload ähnlich + + +--- Antworten: Claudia/Heike 21.09. ----------------- +- Released for Inter/Intra raus [done] +- Wenn License Holder fehlt, AH GmbH setzen und auf separate Liste [done] +- Usage Rights -> immer restricted [done] +- Englisch OK +- Additional Info in einzelnen Zeilen [done] +- Comment und Description [done] +- Status: Liste mit Status != ACCEPTED an Claudia/Heike + Archiv ist nur andere Kategorie +- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess +- Seite 2 als Preview +- Filename Preview nach Feld fileName setzen +----------------------------------------------------- + -- Timeline - - Fertigstellung Migrations-Programm - - Download/Upload Metadaten - - Download/Upload Dateien CREATE TABLE Record ( @@ -46,6 +70,8 @@ CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'char DROP ALL OBJECTS +jdbc:h2:C:\Temp\Migration AH\Entry Export\entries + CREATE TABLE Entry ( ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, EntryId INTEGER NOT NULL, @@ -60,7 +86,7 @@ AS SELECT MediaType, Filesize, Json -FROM CSVREAD('C:/Temp/Migration AH/migration.log', 'EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json', 'charset=UTF-8 fieldSeparator=|~|'); +FROM CSVREAD('C:\Temp\Migration AH\Entry Export\migration.csv', null, 'charset=UTF-8 fieldSeparator=|~|'); SELECT * FROM Entry; diff --git a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java index 5ffd88d..0a2a624 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java @@ -25,7 +25,8 @@ public class Definitions { classifications.put("Country.Global", new Classification("dff7666b-326b-4e1c-912c-a7b4006c8500", 1)); // /Definitions/Animal Health/Country classifications.put("Agencies.Bayer AH", new Classification("8de5ef0e-3c6d-451c-b82b-a7b4006c954c", 1)); // /Definitions/Agencies/Bayer - classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1)); // Alles global/unrestricted? + classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1)); + classifications.put("Usage.Restricted", new Classification("59cf873fe1be4dd9ae8ba73100a2bd89", 1)); // /Bayer/Animal Health/BusinessUnit classifications.put("BusinessUnit.CAP", new Classification("306e050a-dd58-46ad-93a7-a73100a2547f", 1)); @@ -37,7 +38,7 @@ public class Definitions { classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1)); classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1)); - classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin + classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); fieldIDs = new HashMap(); fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b"); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java index b0fc339..ca7bb8e 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java @@ -17,7 +17,7 @@ import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entrylist; import com.fasterxml.jackson.databind.ObjectMapper; -@SpringBootApplication +//@SpringBootApplication @PropertySource("file:secret.properties") public class EntryDownloader implements CommandLineRunner { diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java new file mode 100644 index 0000000..7c40e36 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java @@ -0,0 +1,65 @@ +package com.bayer.edam.migration; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bayer.edam.migration.mam.Entry; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class EntryProcessor { + + private static final Logger log = LoggerFactory.getLogger(EntryProcessor.class); + + static final String fieldSeparator = "|~|"; + + private static final File entriesFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv"); + + private ObjectMapper jsonMapper = new ObjectMapper(); + + public EntryProcessor() throws IOException { + LineIterator iter = FileUtils.lineIterator(entriesFile, "UTF-8"); + + // skip header + iter.next(); + + HashSet orgSet = new HashSet(); + + while (iter.hasNext()) { + String line = iter.next(); + line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length()); + Entry entry = jsonMapper.readValue(line, Entry.class); + if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) { + + String orgId = null; + + if (entry.copyright != null && entry.copyright.organization != null) { + orgId = entry.copyright.organization.get("id"); + orgSet.add(orgId); + } + + + log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId); + } + } + + for (String orgId : orgSet) { + log.info(orgId); + } + + } + + public static void main(String[] args) { + try { + new EntryProcessor(); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 5dd836e..47db4da 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -109,7 +109,7 @@ public class MamMigrationSelective implements CommandLineRunner { record.addClassification("MediaAssistant-Migration"); record.addClassification("Country.Global"); record.addClassification("Agencies.Bayer AH"); - record.addClassification("Usage.Unrestricted"); + record.addClassification("Usage.Restricted"); // status if (entry.stateId == 6) { // ACCEPTED @@ -146,9 +146,9 @@ public class MamMigrationSelective implements CommandLineRunner { if (entry.comment.get("en") != null) description += "Comment: " + entry.comment.get("en") + "\n\n"; - String migrationDescription = "Migrated from Media Assistant ID " + entry.id + ", Date of Issue: " + String migrationDescription = "Migrated from Media Assistant ID " + entry.id + "\nDate of Issue: " + stringDate.format(entry.creationDate); - description += "[" + migrationDescription + "]"; + description += migrationDescription; record.addField("byr_ah_description", description); // Keywords @@ -159,12 +159,18 @@ public class MamMigrationSelective implements CommandLineRunner { // License Information record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder)); + + // SR: If license holder is not "Bayer Animal Health GmbH" put on review list + if (!entry.copyright.organization.get("id").equals("3740")) { + log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)"); + } + + record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse)); String licenseDescription = ""; if (entry.copyright.use != null) - licenseDescription += entry.copyright.use + "\n"; - licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet"); + licenseDescription += entry.copyright.use; record.addField("byr_ah_licenseComments", licenseDescription); if (entry.gpc != null) diff --git a/AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java b/AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java new file mode 100644 index 0000000..278a3b5 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java @@ -0,0 +1,44 @@ +package com.bayer.edam.migration.attic; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LogFileCleaner { + + private static final Logger log = LoggerFactory.getLogger(LogFileCleaner.class); + private static final String[] files = { "C:\\Temp\\Migration AH\\Entry Export\\migration.log", + "C:\\Temp\\Migration AH\\Entry Export\\migration.log.1", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.2", + "C:\\Temp\\Migration AH\\Entry Export\\migration.log.3", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.4" }; + private static final File outputFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv"); + + public LogFileCleaner() { + try { + log.info("Write CSV header"); + Charset encoding = Charset.forName("UTF-8"); + FileUtils.writeStringToFile(outputFile, "EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json", encoding, true); + + for (String filename : files) { + log.info("Reading file " + filename); + + List lines = FileUtils.readLines(new File(filename), "UTF-8"); + for (String line : lines) { + line = line.substring(90) + "\n"; + FileUtils.writeStringToFile(outputFile, line, encoding, true); + } + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + public static void main(String[] args) { + new LogFileCleaner(); + } + +}