progress

AH MAM Migration/pom.xml

@@ -38,6 +38,11 @@
 			<version>2.4.1</version>
 			<type>jar</type>
 		</dependency>
+		<dependency>
+			<groupId>commons-io</groupId>
+			<artifactId>commons-io</artifactId>
+			<version>2.5</version>
+		</dependency>
 	</dependencies>
 
 	<build>

AH MAM Migration/questions.txt

@@ -1,28 +1,32 @@
 - English only?
 - CreationDate ist read-only
 - Mapping Media Type
-- Wer soll als byr_ah_projectLead gesetzt werden?
+- Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder?
+- Soll muss der MAM-Uploader irgendwo gespeichert werden?
 - "Bayer Animal Health GmbH" immer als License Holder OK?
 - "comment" mit in description?
 - MAM-Status "ARCHIVED" --> eDAM.Archived?
 - Was ist mit Status != ACCEPTED or ARCHIVED?
 - Regelung Cutover: Kein Upload mehr ab KW42 (16.10.)
 - Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763)
+- Beispiel für ausgefüllte GPC-Daten
+- was ist mit regionalLicenseComment/personalLicenseComment?
+- Wird GPC-Block verwendet?
+- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = Retire Date?
+- GPC No. = LMR ID?
 
 
-CREATE TABLE House (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, date DATE NOT NULL, description VARCHAR(255));
+CREATE TABLE Record (
---CREATE TABLE Room (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, qm INTEGER NOT NULL, HouseID INTEGER NOT NULL, FOREIGN KEY(HouseID) REFERENCES House (ID));
-INSERT INTO House (date, description) VALUES ('2017-09-18', 'Das blaue Haus');
-INSERT INTO House (date, description) VALUES ('2017-09-15', 'Das grüne Haus');
-CALL CSVWRITE ('C:/Temp/Migration AH/house.csv', 'SELECT * FROM House','charset=UTF-8');
-DROP TABLE House;
-CREATE TABLE House (
   ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY,
-  date DATE NOT NULL,
+  Location VARCHAR NOT NULL,
-  description VARCHAR(255))
+  Entry VARCHAR NOT NULL)
 AS SELECT
-  ID,
+  null,
-  date,
+  Location,
-  description
+  Entry
-FROM CSVREAD('C:/Temp/Migration AH/house.csv', null, 'charset=UTF-8');
+FROM CSVREAD('C:/Temp/Migration AH/entry.log', null, 'charset=UTF-8 fieldSeparator=|~|');
-SELECT * FROM House;
+
+SELECT * FROM Record;
+
+CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'charset=UTF-8');
+

AH MAM Migration/src/com/bayer/edam/migration/Definitions.java

@@ -37,16 +37,19 @@ public class Definitions {
 		classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1));
 		classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1));
 		
-		classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // Mapping von Martin
+		classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin
 		
 		fieldIDs = new HashMap<String, String>();
 		fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b");
+		fieldIDs.put("byr_ah_approvalNumber", "69d65b63-613b-49d6-bc11-a731010bbec8");
 		fieldIDs.put("byr_ah_description", "f9e40f87-493e-4f17-b129-a6dd00d063b2"); // Additional Information
 		fieldIDs.put("byr_ah_projectLead", "5400d446-9d95-4158-ac06-a6dd00d06a12");
-		fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c");
+		fieldIDs.put("byr_ah_brandCommunicationRepresentative", "5b34f2db-efff-48fd-bc7f-a731010bbda1");
-		fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
 		fieldIDs.put("byr_creationDate", "403fea1c-c154-4e3b-a941-a68a00d4fcc3");
 		fieldIDs.put("byr_ah_keywords_tl", "6d582cfaa5b94101ab1fa6fd007e2fcd");
+		fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c");
+		fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
+		fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a");
 	}
 	
 	public static Field createSingleValueField(String name, String value) {

AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java

@@ -1,6 +1,8 @@
 package com.bayer.edam.migration;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -16,20 +18,37 @@ public class LoggingRequestInterceptor implements ClientHttpRequestInterceptor {
 	final static Logger log = LoggerFactory.getLogger(LoggingRequestInterceptor.class);
 
 	@Override
-	public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution)
+	public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException {
-			throws IOException {
 		traceRequest(request, body);
 		ClientHttpResponse response = execution.execute(request, body);
+//		traceResponse(response);
 		return response;
 	}
 
 	private void traceRequest(HttpRequest request, byte[] body) throws IOException {
-		log.debug("===========================request begin================================================");
+		log.debug("===================request begin===================");
 		log.debug("URI         : {}", request.getURI());
 		log.debug("Method      : {}", request.getMethod());
 		log.debug("Headers     : {}", request.getHeaders());
 		log.debug("Request body: {}", new String(body, "UTF-8"));
-		log.debug("==========================request end================================================");
+		log.debug("====================request end====================");
+	}
+
+	private void traceResponse(ClientHttpResponse response) throws IOException {
+		StringBuilder inputStringBuilder = new StringBuilder();
+		BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody(), "UTF-8"));
+		String line = bufferedReader.readLine();
+		while (line != null) {
+			inputStringBuilder.append(line);
+			inputStringBuilder.append('\n');
+			line = bufferedReader.readLine();
+		}
+		log.debug("============================response begin==========================================");
+		log.debug("Status code  : {}", response.getStatusCode());
+		log.debug("Status text  : {}", response.getStatusText());
+		log.debug("Headers      : {}", response.getHeaders());
+		log.debug("Response body: {}", inputStringBuilder.toString());
+		log.debug("=======================response end=================================================");
 	}
 
 }

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -1,6 +1,8 @@
 package com.bayer.edam.migration;
 
+import java.io.File;
 import java.net.URI;
+import java.nio.charset.Charset;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -9,6 +11,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -29,6 +32,7 @@ import org.springframework.web.client.RestTemplate;
 import com.bayer.edam.migration.adam.Token;
 import com.bayer.edam.migration.adam.WriteRecord;
 import com.bayer.edam.migration.mam.Entry;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 @SpringBootApplication
 @PropertySource("file:secret.properties")
@@ -36,8 +40,15 @@ public class MamMigrationSelective implements CommandLineRunner {
 
 	private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
 	
-	static String adamHost = "https://edam-q.bayer.com:2015";
+	static final String adamHost = "https://edam-q.bayer.com:2015";
-	static int[] selection = {   71948,   71048,70778/*,70746,70142,70361,63924,71263,63201,56942,57641,70745,70440,70340,70288,71725*/};
+	static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
+	static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
+			70288, 71725*/ };
+
+	static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log");
+	static final File problemLog = new File("C:\\Temp\\Migration AH\\problem.log");
+	static final String fieldSeparator = "|~|";
+	static final String lineSeparator = "\r\n";
 
 	@Value("${mam.auth}")
 	private String mamAuth;
@@ -52,11 +63,17 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private static SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
 	private static SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
 
+	private static ObjectMapper jsonMapper = new ObjectMapper();
+
 	private List<String> locations = new ArrayList<String>();
 
 	@Override
 	public void run(String... strings) throws Exception {
 
+		log.info("Write log headers");
+		FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
+		FileUtils.writeStringToFile(problemLog, "entryId" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
+
 		restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
 		List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
 		interceptors.add(new LoggingRequestInterceptor());
@@ -70,28 +87,24 @@ public class MamMigrationSelective implements CommandLineRunner {
 		for (int id : selection) {
 			log.info("GET entry " + id);
 
-			Entry entry = restTemplate
+			Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
-					.exchange("https://service.media-assistant.animalhealth.bayer.com/rest/migration/entry/{id}",
-							HttpMethod.GET, entity, Entry.class, id)
-					.getBody();
-
-			// log.info(entry.toFullString());
 
 			postToAdam(entry);
 		}
 		
-		// delete all created records
+//		// delete all created records
 //		for (String location : locations) {
-//			ResponseEntity<String> response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE,
+//			ResponseEntity<String> response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE, adamHeadersEntity,
-//					adamHeadersEntity, String.class);
+//					String.class);
 //			log.info("Deleted " + location + ": " + response.toString());
 //		}
 
 		invalidateAdamToken();
 	}
 
-	void postToAdam(Entry entry) {
+	void postToAdam(Entry entry) throws Exception {
 		WriteRecord record = new WriteRecord();
+		String entryJson = jsonMapper.writeValueAsString(entry);
 
 		// globally applicable classifications
 		record.addClassification("MediaAssistant-Migration");
@@ -106,6 +119,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 			record.addClassification("Status.Archived");
 		} else {
 			log.error("Skipping entry " + entry.id + ": No target status for stateId == " + entry.stateId);
+			FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
 			return;
 		}
 
@@ -113,13 +127,15 @@ public class MamMigrationSelective implements CommandLineRunner {
 		record.addClassification("BusinessUnit.CAP");
 
 		// media type
+		// TODO
 		record.addClassification("MediaType.Other");
 
 		// Fields
 		record.addField("byr_Title", entry.name.get("en"));
+		
+		// TODO
 		record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
-		record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
+		record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
-		record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
 
 		// Description / Additional Information
 		String description = entry.description.get("en");
@@ -136,25 +152,39 @@ public class MamMigrationSelective implements CommandLineRunner {
 		description += "[" + migrationDescription + "]";
 		record.addField("byr_ah_description", description);
 
-		// Media Type
-		// TODO
-
 		// Keywords
 		if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
 			Set<String> keywords = new HashSet<String>(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
 			record.addField("byr_ah_keywords_tl", new ArrayList<String>(keywords));
 		}
 		
+		// License Information
+		record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
+		record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
+		
+		String licenseDescription = "";
+		if (entry.copyright.use != null)
+			licenseDescription +=  entry.copyright.use + "\n";
+		licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet");
+		record.addField("byr_ah_licenseComments", licenseDescription);
+		
+		if (entry.gpc != null)
+			record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); // TODO
+
 		HttpEntity<WriteRecord> httpEntity = new HttpEntity<WriteRecord>(record, adamHeaders);
 		try {
 			URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class);
 			log.info("Created " + location);
 
+			FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
+					Charset.forName("utf-8"), true);
+
 			locations.add(location.toString());
 
 		} catch (HttpStatusCodeException hsce) {
-			log.error(hsce.getStatusCode().toString());
+			log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
 			log.error(hsce.getResponseBodyAsString());
+			FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
 		}
 	}
 
@@ -175,7 +205,8 @@ public class MamMigrationSelective implements CommandLineRunner {
 		headers.add("Accept", "application/hal+json");
 		headers.add("Content-Type", "application/json");
 		headers.add("Authorization", "Basic " + adamAuth);
-		headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
+		headers.add("set-immediateSearchIndexUpdate", "true"); // see
+																// https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
 		HttpEntity<String> entity = new HttpEntity<String>("", headers);
 		Token token = restTemplate.exchange(adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody();
 
@@ -190,21 +221,16 @@ public class MamMigrationSelective implements CommandLineRunner {
 	}
 
 	private void invalidateAdamToken() {
-		ResponseEntity<String> response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE,
+		ResponseEntity<String> response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
-				adamHeadersEntity, String.class);
 		log.info("Logged out from adam API: " + response.toString());
 	}
 
 	/*
-	 * // Quoted "Z" to indicate UTC, no timezone offset private static
+	 * // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new
-	 * SimpleDateFormat outputDateFormat = new
+	 * SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-	 * SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat
-	 * inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
 	 * 
-	 * public static String toIso8601(String isoDate) { try { return
+	 * public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
-	 * outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
+	 * (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; }
-	 * (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return
-	 * null; }
 	 */
 
 	public static void main(String args[]) {

AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java

@@ -1,28 +1,17 @@
 package com.bayer.edam.migration.mam;
 
-import java.lang.reflect.Field;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
 /**
  * Representation of /rest/migration/entry
  */
 public class Entry {
 
-	private static final Logger log = LoggerFactory.getLogger(Entry.class);
-
 	public static class Type {
 		public Long id;
 		public Map<String, String> name;
-
-		public Type() {
-		}
 	}
 
 	public static class Copyright {
@@ -43,8 +32,17 @@ public class Entry {
 		public Boolean regionalLicense;
 		public Boolean personalLicense;
 
-		public Copyright() {
+	}
-		}
+	
+	public static class Gpc {
+		public Long id;
+		public Map<String, String> owner;
+		public Map<String, String> organization;
+		public String gpcState;
+		public String gpcNumber;
+		public Date expirationDate;
+		public String lastNotification;
+		public Integer gpcStateId;
 	}
 
 	public static class Download {
@@ -58,15 +56,13 @@ public class Entry {
 		public Map<String, String> description;
 		public String fileName;
 		public String fileNameOnDisc;
-
-		public Download() {
-		}
 	}
 
 	public Long id;
 	public String href;
 	public Long fileSize;
 	public String fileName;
+	public String internalKey;
 	public Date creationDate;
 	public Date uploadDate;
 	public Map<String, String> name;
@@ -77,14 +73,14 @@ public class Entry {
 	public Map<String, String> uploader;
 	public Map<String, String> indexer;
 	public Double ratingValue;
-	public Map<String, String> release;
+	public Map<String, String> release; // ??
 	public String brandingNet;
 	public Type mediaType;
 	public Copyright copyright;
 	public String master;
 	public Map<String, String> project;
 	public Type imageType;
-	// public String gpc;
+	public Gpc gpc;
 	public String mediaAssistantId;
 	public Map<String, String> metaData;
 	public List<String> categories;
@@ -93,42 +89,39 @@ public class Entry {
 	public List<Download> downloadList;
 	public List<Download> downloadListPublicFolder;
 
-	public Entry() {
-	}
-
 	@Override
 	public String toString() {
 		return "Entry{" + "id=" + id + ", name=" + name.get("en") + ", creationDate=" + creationDate + ", user="
 				+ user.get("id") + ", copyright.organization.id=" + copyright.organization.get("id") + '}';
 	}
 
-	public String toFullString() {
+//	public String toFullString() {
-		StringBuilder result = new StringBuilder();
+//		StringBuilder result = new StringBuilder();
-		String newLine = System.getProperty("line.separator");
+//		String newLine = System.getProperty("line.separator");
-
+//
-		result.append(this.getClass().getName());
+//		result.append(this.getClass().getName());
-		result.append(" Object {");
+//		result.append(" Object {");
-		result.append(newLine);
+//		result.append(newLine);
-
+//
-		// determine fields declared in this class only (no fields of superclass)
+//		// determine fields declared in this class only (no fields of superclass)
-		Field[] fields = this.getClass().getDeclaredFields();
+//		Field[] fields = this.getClass().getDeclaredFields();
-
+//
-		// print field names paired with their values
+//		// print field names paired with their values
-		for (Field field : fields) {
+//		for (Field field : fields) {
-			result.append("  ");
+//			result.append("  ");
-			try {
+//			try {
-				result.append(field.getName());
+//				result.append(field.getName());
-				result.append(": ");
+//				result.append(": ");
-				// requires access to private field:
+//				// requires access to private field:
-				result.append(field.get(this));
+//				result.append(field.get(this));
-			} catch (IllegalAccessException ex) {
+//			} catch (IllegalAccessException ex) {
-				log.error(ex.toString());
+//				log.error(ex.toString());
-			}
+//			}
-			result.append(newLine);
+//			result.append(newLine);
-		}
+//		}
-		result.append("}");
+//		result.append("}");
-
+//
-		return result.toString();
+//		return result.toString();
-	}
+//	}
 
 }