EntryProcessor

AH MAM Migration/questions.txt

@@ -1,30 +1,54 @@
+1. Status
+- Übertragen der Datensätze funktioniert grundsätzlich
+- MAM -> JSON (komme darauf zurück) -> eDAM
+- Felder da, suchbar
+- Es fehlen noch die verlinkten Strukturen: Kategorien, Benutzer
+
+2. Fragen
 - English only?
 - CreationDate ist read-only
 - Mapping Media Type --> bringt Martin mit
 - Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder?
-- Soll muss der MAM-Uploader irgendwo gespeichert werden?
+- Soll muss der MAM-Uploader irgendwo gespeichert werden? --> Additional Info
 - "Bayer Animal Health GmbH" immer als License Holder OK?
 - "comment" mit in description?
 - MAM-Status "ARCHIVED" --> eDAM.Archived?
 - Was ist mit Status != ACCEPTED or ARCHIVED?
 - Regelung Cutover: Kein Upload mehr ab KW42 (16.10.)
 
-License:
+  License:
 - Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763) --> mandatory
 - release.forInternet/forIntranet --> License comments
 - GPC owner/e-mail?
 
-GPC:
+  GPC:
 - Beispiel für ausgefüllte GPC-Daten/Wird GPC-Block verwendet?
 - was ist mit regionalLicenseComment/personalLicenseComment?
 - Ist "Date of Issue" Release Date? Ist gpc.expirationDate = LMR Expiration Date?
 - GPC No. = LMR ID?
 
+3. Timeline
+- Fertigstellung Migrations-Programm
+- Download/Upload Metadaten
+- Download/Upload Dateien
+- Download = knapp 4 Tage, Upload ähnlich
+
+
+--- Antworten: Claudia/Heike 21.09. -----------------
+- Released for Inter/Intra raus [done]
+- Wenn License Holder fehlt, AH GmbH setzen und auf separate Liste [done]
+- Usage Rights -> immer restricted [done]
+- Englisch OK
+- Additional Info in einzelnen Zeilen [done]
+- Comment und Description [done]
+- Status: Liste mit Status != ACCEPTED an Claudia/Heike
+  Archiv ist nur andere Kategorie
+- Agency: Print Supplier (wenn gesetzt, sonst Bayer AH) --> Martin klärt Prozess
+- Seite 2 als Preview
+- Filename Preview nach Feld fileName setzen
+-----------------------------------------------------
+
 
-- Timeline
-  - Fertigstellung Migrations-Programm
-  - Download/Upload Metadaten
-  - Download/Upload Dateien
 
 
 CREATE TABLE Record (
@@ -46,6 +70,8 @@ CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'char
 DROP ALL OBJECTS
 
 
+jdbc:h2:C:\Temp\Migration AH\Entry Export\entries
+
 CREATE TABLE Entry (
   ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY,
   EntryId INTEGER NOT NULL,
@@ -60,7 +86,7 @@ AS SELECT
   MediaType,
   Filesize,
   Json
-FROM CSVREAD('C:/Temp/Migration AH/migration.log', 'EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json', 'charset=UTF-8 fieldSeparator=|~|');
+FROM CSVREAD('C:\Temp\Migration AH\Entry Export\migration.csv', null, 'charset=UTF-8 fieldSeparator=|~|');
 
 SELECT * FROM Entry;
 

AH MAM Migration/src/com/bayer/edam/migration/Definitions.java

@@ -25,7 +25,8 @@ public class Definitions {
 		
 		classifications.put("Country.Global", new Classification("dff7666b-326b-4e1c-912c-a7b4006c8500", 1)); // /Definitions/Animal Health/Country
 		classifications.put("Agencies.Bayer AH", new Classification("8de5ef0e-3c6d-451c-b82b-a7b4006c954c", 1)); // /Definitions/Agencies/Bayer
-		classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1)); // Alles global/unrestricted?
+		classifications.put("Usage.Unrestricted", new Classification("6b95a635-40e9-4782-ad5f-a73100a2bb94", 1));
+		classifications.put("Usage.Restricted", new Classification("59cf873fe1be4dd9ae8ba73100a2bd89", 1));
 		
 		// /Bayer/Animal Health/BusinessUnit
 		classifications.put("BusinessUnit.CAP", new Classification("306e050a-dd58-46ad-93a7-a73100a2547f", 1));
@@ -37,7 +38,7 @@ public class Definitions {
 		classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1));
 		classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1));
 		
-		classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin
+		classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1));
 		
 		fieldIDs = new HashMap<String, String>();
 		fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b");

AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java

@@ -17,7 +17,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
 import com.bayer.edam.migration.mam.Entrylist;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-@SpringBootApplication
+//@SpringBootApplication
 @PropertySource("file:secret.properties")
 public class EntryDownloader implements CommandLineRunner {
 	

AH MAM Migration/src/com/bayer/edam/migration/EntryProcessor.java

@@ -0,0 +1,65 @@
+package com.bayer.edam.migration;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.bayer.edam.migration.mam.Entry;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class EntryProcessor {
+
+	private static final Logger log = LoggerFactory.getLogger(EntryProcessor.class);
+
+	static final String fieldSeparator = "|~|";
+
+	private static final File entriesFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv");
+
+	private ObjectMapper jsonMapper = new ObjectMapper();
+
+	public EntryProcessor() throws IOException {
+		LineIterator iter = FileUtils.lineIterator(entriesFile, "UTF-8");
+
+		// skip header
+		iter.next();
+		
+		HashSet<String> orgSet = new HashSet<String>();
+
+		while (iter.hasNext()) {
+			String line = iter.next();
+			line = line.substring(line.lastIndexOf(fieldSeparator) + fieldSeparator.length());
+			Entry entry = jsonMapper.readValue(line, Entry.class);
+			if (entry.copyright == null || entry.copyright.organization == null || !entry.copyright.organization.get("id").equals("3740")) {
+				
+				String orgId = null;
+				
+				if (entry.copyright != null && entry.copyright.organization != null) {
+					orgId = entry.copyright.organization.get("id");
+					orgSet.add(orgId);
+				}
+				
+				
+				log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' " + orgId);
+			}
+		}
+		
+		for (String orgId : orgSet) {
+			log.info(orgId);
+		}
+
+	}
+
+	public static void main(String[] args) {
+		try {
+			new EntryProcessor();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+}

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -109,7 +109,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 		record.addClassification("MediaAssistant-Migration");
 		record.addClassification("Country.Global");
 		record.addClassification("Agencies.Bayer AH");
-		record.addClassification("Usage.Unrestricted");
+		record.addClassification("Usage.Restricted");
 
 		// status
 		if (entry.stateId == 6) { // ACCEPTED
@@ -146,9 +146,9 @@ public class MamMigrationSelective implements CommandLineRunner {
 		if (entry.comment.get("en") != null)
 			description += "Comment: " + entry.comment.get("en") + "\n\n";
 
-		String migrationDescription = "Migrated from Media Assistant ID " + entry.id + ", Date of Issue: "
+		String migrationDescription = "Migrated from Media Assistant ID " + entry.id + "\nDate of Issue: "
 				+ stringDate.format(entry.creationDate);
-		description += "[" + migrationDescription + "]";
+		description += migrationDescription;
 		record.addField("byr_ah_description", description);
 
 		// Keywords
@@ -159,12 +159,18 @@ public class MamMigrationSelective implements CommandLineRunner {
 		
 		// License Information
 		record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
+		
+		// SR: If license holder is not "Bayer Animal Health GmbH" put on review list
+		if (!entry.copyright.organization.get("id").equals("3740")) {
+			log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
+		}
+		
+		
 		record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
 		
 		String licenseDescription = "";
 		if (entry.copyright.use != null)
-			licenseDescription +=  entry.copyright.use + "\n";
+			licenseDescription +=  entry.copyright.use;
-		licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet");
 		record.addField("byr_ah_licenseComments", licenseDescription);
 		
 		if (entry.gpc != null)

AH MAM Migration/src/com/bayer/edam/migration/attic/LogFileCleaner.java

@@ -0,0 +1,44 @@
+package com.bayer.edam.migration.attic;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogFileCleaner {
+
+	private static final Logger log = LoggerFactory.getLogger(LogFileCleaner.class);
+	private static final String[] files = { "C:\\Temp\\Migration AH\\Entry Export\\migration.log",
+			"C:\\Temp\\Migration AH\\Entry Export\\migration.log.1", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.2",
+			"C:\\Temp\\Migration AH\\Entry Export\\migration.log.3", "C:\\Temp\\Migration AH\\Entry Export\\migration.log.4" };
+	private static final File outputFile = new File("C:\\Temp\\Migration AH\\Entry Export\\migration.csv");
+
+	public LogFileCleaner() {
+		try {
+			log.info("Write CSV header");
+			Charset encoding = Charset.forName("UTF-8");
+			FileUtils.writeStringToFile(outputFile, "EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json", encoding, true);
+			
+			for (String filename : files) {
+				log.info("Reading file " + filename);
+				
+				List<String> lines = FileUtils.readLines(new File(filename), "UTF-8");
+				for (String line : lines) {
+					line = line.substring(90) + "\n";
+					FileUtils.writeStringToFile(outputFile, line, encoding, true);
+				}
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+	}
+
+	public static void main(String[] args) {
+		new LogFileCleaner();
+	}
+
+}