tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EntryProcessor

Browse Source
This commit is contained in:
gexce
2017-09-21 15:02:59 +02:00
parent a88d58752d
commit 59697c22d7
6 changed files with 158 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/questions.txt
+30 -4
View File
@@ -1,8 +1,15 @@
1. Status
- Übertragen der Datensätze funktioniert grundsätzlich
- MAM -> JSON (komme darauf zurück) -> eDAM
- Felder da, suchbar
- Es fehlen noch die verlinkten Strukturen: Kategorien, Benutzer
2. Fragen
- English only?
- CreationDate ist read-only
- Mapping Media Type --> bringt Martin mit
- Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder?
- Soll muss der MAM-Uploader irgendwo gespeichert werden?
- Soll muss der MAM-Uploader irgendwo gespeichert werden? --> Additional Info
- "Bayer Animal Health GmbH" immer als License Holder OK?
- "comment" mit in description?
- MAM-Status "ARCHIVED" --> eDAM.Archived?
@@ -20,11 +27,28 @@ GPC:
- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = LMR Expiration Date?
- GPC No. = LMR ID?
- Timeline
3. Timeline
- Fertigstellung Migrations-Programm
- Download/Upload Metadaten
- Download/Upload Dateien
- Download = knapp 4 Tage, Upload ähnlich
--- Antworten: Claudia/Heike 21.09. -----------------
- Released for Inter/Intra raus [done]
- Wenn License Holder fehlt, AH GmbH setzen und auf separate Liste [done]
- Usage Rights -> immer restricted [done]
- Englisch OK
- Additional Info in einzelnen Zeilen [done]
- Comment und Description [done]
- Status: Liste mit Status != ACCEPTED an Claudia/Heike
Archiv ist nur andere Kategorie
- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess
- Seite 2 als Preview
- Filename Preview nach Feld fileName setzen
-----------------------------------------------------
CREATE TABLE Record (
@@ -46,6 +70,8 @@ CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'char
DROP ALL OBJECTS
jdbc:h2:C:\Temp\Migration AH\Entry Export\entries
CREATE TABLE Entry (
ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY,
EntryId INTEGER NOT NULL,
@@ -60,7 +86,7 @@ AS SELECT
MediaType,
Filesize,
Json
FROM CSVREAD('C:/Temp/Migration AH/migration.log', 'EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json', 'charset=UTF-8 fieldSeparator=|~|');
FROM CSVREAD('C:\Temp\Migration AH\Entry Export\migration.csv', null, 'charset=UTF-8 fieldSeparator=|~|');
SELECT * FROM Entry;
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+3 -2
View File
@@ -25,7 +25,8 @@ public class Definitions {
classifications.put("Country.Global", new Classification("dff7666b-326b-4e1c-912c-a7b4006c8500", 1)); // /Definitions/Animal Health/Country
classifications.put("Agencies.Bayer AH", new Classification("8de5ef0e-3c6d-451c-b82b-a7b4006c954c", 1)); // /Definitions/Agencies/Bayer
classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1)); // Alles global/unrestricted?
classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1));
classifications.put("Usage.Restricted", new Classification("59cf873fe1be4dd9ae8ba73100a2bd89", 1));
// /Bayer/Animal Health/BusinessUnit
classifications.put("BusinessUnit.CAP", new Classification("306e050a-dd58-46ad-93a7-a73100a2547f", 1));
@@ -37,7 +38,7 @@ public class Definitions {
classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1));
classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1));
classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin
classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1));
fieldIDs = new HashMap<String, String>();
fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b");
AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java
+1 -1
View File
@@ -17,7 +17,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.fasterxml.jackson.databind.ObjectMapper;
@SpringBootApplication
//@SpringBootApplication
@PropertySource("file:secret.properties")
public class EntryDownloader implements CommandLineRunner {
AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java
+65
View File
@@ -0,0 +1,65 @@
package com.bayer.edam.migration;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bayer.edam.migration.mam.Entry;
import com.fasterxml.jackson.databind.ObjectMapper;
public class EntryProcessor {
private static final Logger log = LoggerFactory.getLogger(EntryProcessor.class);
static final String fieldSeparator = "|~|";
private static final File entriesFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv");
private ObjectMapper jsonMapper = new ObjectMapper();
public EntryProcessor() throws IOException {
LineIterator iter = FileUtils.lineIterator(entriesFile, "UTF-8");
// skip header
iter.next();
HashSet<String> orgSet = new HashSet<String>();
while (iter.hasNext()) {
String line = iter.next();
line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
Entry entry = jsonMapper.readValue(line, Entry.class);
if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
String orgId = null;
if (entry.copyright != null && entry.copyright.organization != null) {
orgId = entry.copyright.organization.get("id");
orgSet.add(orgId);
}
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId);
}
}
for (String orgId : orgSet) {
log.info(orgId);
}
}
public static void main(String[] args) {
try {
new EntryProcessor();
} catch (IOException e) {
e.printStackTrace();
}
}
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+11 -5
View File
@@ -109,7 +109,7 @@ public class MamMigrationSelective implements CommandLineRunner {
record.addClassification("MediaAssistant-Migration");
record.addClassification("Country.Global");
record.addClassification("Agencies.Bayer AH");
record.addClassification("Usage.Unrestricted");
record.addClassification("Usage.Restricted");
// status
if (entry.stateId == 6) { // ACCEPTED
@@ -146,9 +146,9 @@ public class MamMigrationSelective implements CommandLineRunner {
if (entry.comment.get("en") != null)
description += "Comment: " + entry.comment.get("en") + "\n\n";
String migrationDescription = "Migrated from Media Assistant ID " + entry.id + ", Date of Issue: "
String migrationDescription = "Migrated from Media Assistant ID " + entry.id + "\nDate of Issue: "
+ stringDate.format(entry.creationDate);
description += "[" + migrationDescription + "]";
description += migrationDescription;
record.addField("byr_ah_description", description);
// Keywords
@@ -159,12 +159,18 @@ public class MamMigrationSelective implements CommandLineRunner {
// License Information
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
// SR: If license holder is not "Bayer Animal Health GmbH" put on review list
if (!entry.copyright.organization.get("id").equals("3740")) {
log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
}
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
String licenseDescription = "";
if (entry.copyright.use != null)
licenseDescription += entry.copyright.use + "\n";
licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet");
licenseDescription += entry.copyright.use;
record.addField("byr_ah_licenseComments", licenseDescription);
if (entry.gpc != null)
AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java
+44
View File
@@ -0,0 +1,44 @@
package com.bayer.edam.migration.attic;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class LogFileCleaner {
private static final Logger log = LoggerFactory.getLogger(LogFileCleaner.class);
private static final String[] files = { "C:\\Temp\\Migration AH\\Entry Export\\migration.log",
"C:\\Temp\\Migration AH\\Entry Export\\migration.log.1", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.2",
"C:\\Temp\\Migration AH\\Entry Export\\migration.log.3", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.4" };
private static final File outputFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv");
public LogFileCleaner() {
try {
log.info("Write CSV header");
Charset encoding = Charset.forName("UTF-8");
FileUtils.writeStringToFile(outputFile, "EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json", encoding, true);
for (String filename : files) {
log.info("Reading file " + filename);
List<String> lines = FileUtils.readLines(new File(filename), "UTF-8");
for (String line : lines) {
line = line.substring(90) + "\n";
FileUtils.writeStringToFile(outputFile, line, encoding, true);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
new LogFileCleaner();
}
}