diff --git a/AH MAM Migration/pom.xml b/AH MAM Migration/pom.xml index 29273d2..689daff 100644 --- a/AH MAM Migration/pom.xml +++ b/AH MAM Migration/pom.xml @@ -38,6 +38,11 @@ 2.4.1 jar + + commons-io + commons-io + 2.5 + diff --git a/AH MAM Migration/questions.txt b/AH MAM Migration/questions.txt index 703e836..1c41c23 100644 --- a/AH MAM Migration/questions.txt +++ b/AH MAM Migration/questions.txt @@ -1,28 +1,32 @@ - English only? - CreationDate ist read-only - Mapping Media Type -- Wer soll als byr_ah_projectLead gesetzt werden? +- Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder? +- Soll muss der MAM-Uploader irgendwo gespeichert werden? - "Bayer Animal Health GmbH" immer als License Holder OK? - "comment" mit in description? - MAM-Status "ARCHIVED" --> eDAM.Archived? - Was ist mit Status != ACCEPTED or ARCHIVED? - Regelung Cutover: Kein Upload mehr ab KW42 (16.10.) - Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763) +- Beispiel für ausgefüllte GPC-Daten +- was ist mit regionalLicenseComment/personalLicenseComment? +- Wird GPC-Block verwendet? +- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = Retire Date? +- GPC No. = LMR ID? -CREATE TABLE House (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, date DATE NOT NULL, description VARCHAR(255)); ---CREATE TABLE Room (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, qm INTEGER NOT NULL, HouseID INTEGER NOT NULL, FOREIGN KEY(HouseID) REFERENCES House (ID)); -INSERT INTO House (date, description) VALUES ('2017-09-18', 'Das blaue Haus'); -INSERT INTO House (date, description) VALUES ('2017-09-15', 'Das grüne Haus'); -CALL CSVWRITE ('C:/Temp/Migration AH/house.csv', 'SELECT * FROM House','charset=UTF-8'); -DROP TABLE House; -CREATE TABLE House ( +CREATE TABLE Record ( ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, - date DATE NOT NULL, - description VARCHAR(255)) + Location VARCHAR NOT NULL, + Entry VARCHAR NOT NULL) AS SELECT - ID, - date, - description -FROM CSVREAD('C:/Temp/Migration AH/house.csv', null, 'charset=UTF-8'); -SELECT * FROM House; + null, + Location, + Entry +FROM CSVREAD('C:/Temp/Migration AH/entry.log', null, 'charset=UTF-8 fieldSeparator=|~|'); + +SELECT * FROM Record; + +CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'charset=UTF-8'); + diff --git a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java index bc2c3b1..5ffd88d 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/Definitions.java @@ -37,16 +37,19 @@ public class Definitions { classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1)); classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1)); - classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // Mapping von Martin + classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin fieldIDs = new HashMap(); fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b"); + fieldIDs.put("byr_ah_approvalNumber", "69d65b63-613b-49d6-bc11-a731010bbec8"); fieldIDs.put("byr_ah_description", "f9e40f87-493e-4f17-b129-a6dd00d063b2"); // Additional Information fieldIDs.put("byr_ah_projectLead", "5400d446-9d95-4158-ac06-a6dd00d06a12"); - fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c"); - fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef"); + fieldIDs.put("byr_ah_brandCommunicationRepresentative", "5b34f2db-efff-48fd-bc7f-a731010bbda1"); fieldIDs.put("byr_creationDate", "403fea1c-c154-4e3b-a941-a68a00d4fcc3"); fieldIDs.put("byr_ah_keywords_tl", "6d582cfaa5b94101ab1fa6fd007e2fcd"); + fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c"); + fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef"); + fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a"); } public static Field createSingleValueField(String name, String value) { diff --git a/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java b/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java index 0359496..368b48a 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java @@ -1,6 +1,8 @@ package com.bayer.edam.migration; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,20 +18,37 @@ public class LoggingRequestInterceptor implements ClientHttpRequestInterceptor { final static Logger log = LoggerFactory.getLogger(LoggingRequestInterceptor.class); @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) - throws IOException { + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { traceRequest(request, body); ClientHttpResponse response = execution.execute(request, body); +// traceResponse(response); return response; } private void traceRequest(HttpRequest request, byte[] body) throws IOException { - log.debug("===========================request begin================================================"); + log.debug("===================request begin==================="); log.debug("URI : {}", request.getURI()); log.debug("Method : {}", request.getMethod()); log.debug("Headers : {}", request.getHeaders()); log.debug("Request body: {}", new String(body, "UTF-8")); - log.debug("==========================request end================================================"); + log.debug("====================request end===================="); + } + + private void traceResponse(ClientHttpResponse response) throws IOException { + StringBuilder inputStringBuilder = new StringBuilder(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody(), "UTF-8")); + String line = bufferedReader.readLine(); + while (line != null) { + inputStringBuilder.append(line); + inputStringBuilder.append('\n'); + line = bufferedReader.readLine(); + } + log.debug("============================response begin=========================================="); + log.debug("Status code : {}", response.getStatusCode()); + log.debug("Status text : {}", response.getStatusText()); + log.debug("Headers : {}", response.getHeaders()); + log.debug("Response body: {}", inputStringBuilder.toString()); + log.debug("=======================response end================================================="); } } \ No newline at end of file diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 268a12b..12f71c8 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -1,6 +1,8 @@ package com.bayer.edam.migration; +import java.io.File; import java.net.URI; +import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -9,6 +11,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -29,15 +32,23 @@ import org.springframework.web.client.RestTemplate; import com.bayer.edam.migration.adam.Token; import com.bayer.edam.migration.adam.WriteRecord; import com.bayer.edam.migration.mam.Entry; +import com.fasterxml.jackson.databind.ObjectMapper; @SpringBootApplication @PropertySource("file:secret.properties") public class MamMigrationSelective implements CommandLineRunner { private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class); + + static final String adamHost = "https://edam-q.bayer.com:2015"; + static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; + static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340, + 70288, 71725*/ }; - static String adamHost = "https://edam-q.bayer.com:2015"; - static int[] selection = { 71948, 71048,70778/*,70746,70142,70361,63924,71263,63201,56942,57641,70745,70440,70340,70288,71725*/}; + static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log"); + static final File problemLog = new File("C:\\Temp\\Migration AH\\problem.log"); + static final String fieldSeparator = "|~|"; + static final String lineSeparator = "\r\n"; @Value("${mam.auth}") private String mamAuth; @@ -51,12 +62,18 @@ public class MamMigrationSelective implements CommandLineRunner { private static SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); - + + private static ObjectMapper jsonMapper = new ObjectMapper(); + private List locations = new ArrayList(); @Override public void run(String... strings) throws Exception { + log.info("Write log headers"); + FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8")); + FileUtils.writeStringToFile(problemLog, "entryId" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8")); + restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory())); List interceptors = new ArrayList(); interceptors.add(new LoggingRequestInterceptor()); @@ -70,28 +87,24 @@ public class MamMigrationSelective implements CommandLineRunner { for (int id : selection) { log.info("GET entry " + id); - Entry entry = restTemplate - .exchange("https://service.media-assistant.animalhealth.bayer.com/rest/migration/entry/{id}", - HttpMethod.GET, entity, Entry.class, id) - .getBody(); + Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); - // log.info(entry.toFullString()); - postToAdam(entry); } - // delete all created records +// // delete all created records // for (String location : locations) { -// ResponseEntity response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE, -// adamHeadersEntity, String.class); +// ResponseEntity response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE, adamHeadersEntity, +// String.class); // log.info("Deleted " + location + ": " + response.toString()); // } invalidateAdamToken(); } - void postToAdam(Entry entry) { + void postToAdam(Entry entry) throws Exception { WriteRecord record = new WriteRecord(); + String entryJson = jsonMapper.writeValueAsString(entry); // globally applicable classifications record.addClassification("MediaAssistant-Migration"); @@ -106,6 +119,7 @@ public class MamMigrationSelective implements CommandLineRunner { record.addClassification("Status.Archived"); } else { log.error("Skipping entry " + entry.id + ": No target status for stateId == " + entry.stateId); + FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true); return; } @@ -113,13 +127,15 @@ public class MamMigrationSelective implements CommandLineRunner { record.addClassification("BusinessUnit.CAP"); // media type + // TODO record.addClassification("MediaType.Other"); // Fields record.addField("byr_Title", entry.name.get("en")); + + // TODO record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead)); - record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder)); - record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse)); + record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead)); // Description / Additional Information String description = entry.description.get("en"); @@ -136,25 +152,39 @@ public class MamMigrationSelective implements CommandLineRunner { description += "[" + migrationDescription + "]"; record.addField("byr_ah_description", description); - // Media Type - // TODO - // Keywords if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) { Set keywords = new HashSet(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"))); record.addField("byr_ah_keywords_tl", new ArrayList(keywords)); } + + // License Information + record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder)); + record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse)); + + String licenseDescription = ""; + if (entry.copyright.use != null) + licenseDescription += entry.copyright.use + "\n"; + licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet"); + record.addField("byr_ah_licenseComments", licenseDescription); + + if (entry.gpc != null) + record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); // TODO HttpEntity httpEntity = new HttpEntity(record, adamHeaders); try { URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class); log.info("Created " + location); - + + FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator, + Charset.forName("utf-8"), true); + locations.add(location.toString()); - + } catch (HttpStatusCodeException hsce) { - log.error(hsce.getStatusCode().toString()); + log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString()); log.error(hsce.getResponseBodyAsString()); + FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true); } } @@ -175,7 +205,8 @@ public class MamMigrationSelective implements CommandLineRunner { headers.add("Accept", "application/hal+json"); headers.add("Content-Type", "application/json"); headers.add("Authorization", "Basic " + adamAuth); - headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push + headers.add("set-immediateSearchIndexUpdate", "true"); // see + // https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push HttpEntity entity = new HttpEntity("", headers); Token token = restTemplate.exchange(adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody(); @@ -190,21 +221,16 @@ public class MamMigrationSelective implements CommandLineRunner { } private void invalidateAdamToken() { - ResponseEntity response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE, - adamHeadersEntity, String.class); + ResponseEntity response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class); log.info("Logged out from adam API: " + response.toString()); } /* - * // Quoted "Z" to indicate UTC, no timezone offset private static - * SimpleDateFormat outputDateFormat = new - * SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat - * inputDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + * // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new + * SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd"); * - * public static String toIso8601(String isoDate) { try { return - * outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch - * (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return - * null; } + * public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch + * (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; } */ public static void main(String args[]) { diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java index 1529982..24e813a 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java @@ -1,28 +1,17 @@ package com.bayer.edam.migration.mam; -import java.lang.reflect.Field; import java.util.Date; import java.util.List; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonCreator; - /** * Representation of /rest/migration/entry */ public class Entry { - private static final Logger log = LoggerFactory.getLogger(Entry.class); - public static class Type { public Long id; public Map name; - - public Type() { - } } public static class Copyright { @@ -43,8 +32,17 @@ public class Entry { public Boolean regionalLicense; public Boolean personalLicense; - public Copyright() { - } + } + + public static class Gpc { + public Long id; + public Map owner; + public Map organization; + public String gpcState; + public String gpcNumber; + public Date expirationDate; + public String lastNotification; + public Integer gpcStateId; } public static class Download { @@ -58,15 +56,13 @@ public class Entry { public Map description; public String fileName; public String fileNameOnDisc; - - public Download() { - } } public Long id; public String href; public Long fileSize; public String fileName; + public String internalKey; public Date creationDate; public Date uploadDate; public Map name; @@ -77,14 +73,14 @@ public class Entry { public Map uploader; public Map indexer; public Double ratingValue; - public Map release; + public Map release; // ?? public String brandingNet; public Type mediaType; public Copyright copyright; public String master; public Map project; public Type imageType; - // public String gpc; + public Gpc gpc; public String mediaAssistantId; public Map metaData; public List categories; @@ -93,42 +89,39 @@ public class Entry { public List downloadList; public List downloadListPublicFolder; - public Entry() { - } - @Override public String toString() { return "Entry{" + "id=" + id + ", name=" + name.get("en") + ", creationDate=" + creationDate + ", user=" + user.get("id") + ", copyright.organization.id=" + copyright.organization.get("id") + '}'; } - public String toFullString() { - StringBuilder result = new StringBuilder(); - String newLine = System.getProperty("line.separator"); - - result.append(this.getClass().getName()); - result.append(" Object {"); - result.append(newLine); - - // determine fields declared in this class only (no fields of superclass) - Field[] fields = this.getClass().getDeclaredFields(); - - // print field names paired with their values - for (Field field : fields) { - result.append(" "); - try { - result.append(field.getName()); - result.append(": "); - // requires access to private field: - result.append(field.get(this)); - } catch (IllegalAccessException ex) { - log.error(ex.toString()); - } - result.append(newLine); - } - result.append("}"); - - return result.toString(); - } +// public String toFullString() { +// StringBuilder result = new StringBuilder(); +// String newLine = System.getProperty("line.separator"); +// +// result.append(this.getClass().getName()); +// result.append(" Object {"); +// result.append(newLine); +// +// // determine fields declared in this class only (no fields of superclass) +// Field[] fields = this.getClass().getDeclaredFields(); +// +// // print field names paired with their values +// for (Field field : fields) { +// result.append(" "); +// try { +// result.append(field.getName()); +// result.append(": "); +// // requires access to private field: +// result.append(field.get(this)); +// } catch (IllegalAccessException ex) { +// log.error(ex.toString()); +// } +// result.append(newLine); +// } +// result.append("}"); +// +// return result.toString(); +// } }