tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
gexce
2017-09-19 17:08:25 +02:00
parent 70bd8f3443
commit 9d8699004a
6 changed files with 153 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/pom.xml
+5
View File
@@ -38,6 +38,11 @@
<version>2.4.1</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
</dependencies>
<build>
AH MAM Migration/questions.txt
+19 -15
View File
@@ -1,28 +1,32 @@
- English only?
- CreationDate ist read-only
- Mapping Media Type
- Wer soll als byr_ah_projectLead gesetzt werden?
- Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder?
- Soll muss der MAM-Uploader irgendwo gespeichert werden?
- "Bayer Animal Health GmbH" immer als License Holder OK?
- "comment" mit in description?
- MAM-Status "ARCHIVED" --> eDAM.Archived?
- Was ist mit Status != ACCEPTED or ARCHIVED?
- Regelung Cutover: Kein Upload mehr ab KW42 (16.10.)
- Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763)
- Beispiel für ausgefüllte GPC-Daten
- was ist mit regionalLicenseComment/personalLicenseComment?
- Wird GPC-Block verwendet?
- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = Retire Date?
- GPC No. = LMR ID?
CREATE TABLE House (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, date DATE NOT NULL, description VARCHAR(255));
--CREATE TABLE Room (ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, qm INTEGER NOT NULL, HouseID INTEGER NOT NULL, FOREIGN KEY(HouseID) REFERENCES House (ID));
INSERT INTO House (date, description) VALUES ('2017-09-18', 'Das blaue Haus');
INSERT INTO House (date, description) VALUES ('2017-09-15', 'Das grüne Haus');
CALL CSVWRITE ('C:/Temp/Migration AH/house.csv', 'SELECT * FROM House','charset=UTF-8');
DROP TABLE House;
CREATE TABLE House (
CREATE TABLE Record (
ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY,
date DATE NOT NULL,
description VARCHAR(255))
Location VARCHAR NOT NULL,
Entry VARCHAR NOT NULL)
AS SELECT
ID,
date,
description
FROM CSVREAD('C:/Temp/Migration AH/house.csv', null, 'charset=UTF-8');
SELECT * FROM House;
null,
Location,
Entry
FROM CSVREAD('C:/Temp/Migration AH/entry.log', null, 'charset=UTF-8 fieldSeparator=|~|');
SELECT * FROM Record;
CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'charset=UTF-8');
AH MAM Migration/src/com/bayer/edam/migration/Definitions.java
+6 -3
View File
@@ -37,16 +37,19 @@ public class Definitions {
classifications.put("Status.Released", new Classification("279fce5751584837bd3aa68a00d4f204", 1));
classifications.put("Status.Archived", new Classification("4306204af75442fda297a7b4006b4a87", 1));
classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // Mapping von Martin
classifications.put("MediaType.Other", new Classification("7c13e620-7eb0-4c12-9dfb-a7b4008f19b1", 1)); // TODO Mapping von Martin
fieldIDs = new HashMap<String, String>();
fieldIDs.put("byr_Title", "31f337fa-add6-4e25-a672-a68a00d4ff3b");
fieldIDs.put("byr_ah_approvalNumber", "69d65b63-613b-49d6-bc11-a731010bbec8");
fieldIDs.put("byr_ah_description", "f9e40f87-493e-4f17-b129-a6dd00d063b2"); // Additional Information
fieldIDs.put("byr_ah_projectLead", "5400d446-9d95-4158-ac06-a6dd00d06a12");
fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c");
fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
fieldIDs.put("byr_ah_brandCommunicationRepresentative", "5b34f2db-efff-48fd-bc7f-a731010bbda1");
fieldIDs.put("byr_creationDate", "403fea1c-c154-4e3b-a941-a68a00d4fcc3");
fieldIDs.put("byr_ah_keywords_tl", "6d582cfaa5b94101ab1fa6fd007e2fcd");
fieldIDs.put("byr_ah_licenseHolderList", "de3f83cb50e747228bfea731010bc27c");
fieldIDs.put("byr_ah_licenseAdditionalPermittedUse", "5c9d43a0-bf70-4fb5-8ad4-a731010bc1ef");
fieldIDs.put("byr_ah_licenseComments", "dd4b899f-10b7-45db-a530-a6dd00d0630a");
}
public static Field createSingleValueField(String name, String value) {
AH MAM Migration/src/com/bayer/edam/migration/LoggingRequestInterceptor.java
+23 -4
View File
@@ -1,6 +1,8 @@
package com.bayer.edam.migration;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -16,20 +18,37 @@ public class LoggingRequestInterceptor implements ClientHttpRequestInterceptor {
final static Logger log = LoggerFactory.getLogger(LoggingRequestInterceptor.class);
@Override
public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution)
throws IOException {
public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException {
traceRequest(request, body);
ClientHttpResponse response = execution.execute(request, body);
// traceResponse(response);
return response;
}
private void traceRequest(HttpRequest request, byte[] body) throws IOException {
log.debug("===========================request begin================================================");
log.debug("===================request begin===================");
log.debug("URI : {}", request.getURI());
log.debug("Method : {}", request.getMethod());
log.debug("Headers : {}", request.getHeaders());
log.debug("Request body: {}", new String(body, "UTF-8"));
log.debug("==========================request end================================================");
log.debug("====================request end====================");
}
private void traceResponse(ClientHttpResponse response) throws IOException {
StringBuilder inputStringBuilder = new StringBuilder();
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(response.getBody(), "UTF-8"));
String line = bufferedReader.readLine();
while (line != null) {
inputStringBuilder.append(line);
inputStringBuilder.append('\n');
line = bufferedReader.readLine();
}
log.debug("============================response begin==========================================");
log.debug("Status code : {}", response.getStatusCode());
log.debug("Status text : {}", response.getStatusText());
log.debug("Headers : {}", response.getHeaders());
log.debug("Response body: {}", inputStringBuilder.toString());
log.debug("=======================response end=================================================");
}
}
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+55 -29
View File
@@ -1,6 +1,8 @@
package com.bayer.edam.migration;
import java.io.File;
import java.net.URI;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -9,6 +11,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
@@ -29,6 +32,7 @@ import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.WriteRecord;
import com.bayer.edam.migration.mam.Entry;
import com.fasterxml.jackson.databind.ObjectMapper;
@SpringBootApplication
@PropertySource("file:secret.properties")
@@ -36,8 +40,15 @@ public class MamMigrationSelective implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
static String adamHost = "https://edam-q.bayer.com:2015";
static int[] selection = { 71948, 71048,70778/*,70746,70142,70361,63924,71263,63201,56942,57641,70745,70440,70340,70288,71725*/};
static final String adamHost = "https://edam-q.bayer.com:2015";
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
70288, 71725*/ };
static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log");
static final File problemLog = new File("C:\\Temp\\Migration AH\\problem.log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@Value("${mam.auth}")
private String mamAuth;
@@ -52,11 +63,17 @@ public class MamMigrationSelective implements CommandLineRunner {
private static SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
private static SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private static ObjectMapper jsonMapper = new ObjectMapper();
private List<String> locations = new ArrayList<String>();
@Override
public void run(String... strings) throws Exception {
log.info("Write log headers");
FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
FileUtils.writeStringToFile(problemLog, "entryId" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
interceptors.add(new LoggingRequestInterceptor());
@@ -70,28 +87,24 @@ public class MamMigrationSelective implements CommandLineRunner {
for (int id : selection) {
log.info("GET entry " + id);
Entry entry = restTemplate
.exchange("https://service.media-assistant.animalhealth.bayer.com/rest/migration/entry/{id}",
HttpMethod.GET, entity, Entry.class, id)
.getBody();
// log.info(entry.toFullString());
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
postToAdam(entry);
}
// delete all created records
// // delete all created records
// for (String location : locations) {
// ResponseEntity<String> response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE,
// adamHeadersEntity, String.class);
// ResponseEntity<String> response = restTemplate.exchange(adamHost + location, HttpMethod.DELETE, adamHeadersEntity,
// String.class);
// log.info("Deleted " + location + ": " + response.toString());
// }
invalidateAdamToken();
}
void postToAdam(Entry entry) {
void postToAdam(Entry entry) throws Exception {
WriteRecord record = new WriteRecord();
String entryJson = jsonMapper.writeValueAsString(entry);
// globally applicable classifications
record.addClassification("MediaAssistant-Migration");
@@ -106,6 +119,7 @@ public class MamMigrationSelective implements CommandLineRunner {
record.addClassification("Status.Archived");
} else {
log.error("Skipping entry " + entry.id + ": No target status for stateId == " + entry.stateId);
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
return;
}
@@ -113,13 +127,15 @@ public class MamMigrationSelective implements CommandLineRunner {
record.addClassification("BusinessUnit.CAP");
// media type
// TODO
record.addClassification("MediaType.Other");
// Fields
record.addField("byr_Title", entry.name.get("en"));
// TODO
record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
// Description / Additional Information
String description = entry.description.get("en");
@@ -136,25 +152,39 @@ public class MamMigrationSelective implements CommandLineRunner {
description += "[" + migrationDescription + "]";
record.addField("byr_ah_description", description);
// Media Type
// TODO
// Keywords
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
Set<String> keywords = new HashSet<String>(Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")));
record.addField("byr_ah_keywords_tl", new ArrayList<String>(keywords));
}
// License Information
record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
String licenseDescription = "";
if (entry.copyright.use != null)
licenseDescription += entry.copyright.use + "\n";
licenseDescription += "Released for internet: " + entry.release.get("forInternet") + "\nReleased for intranet: " + entry.release.get("forIntranet");
record.addField("byr_ah_licenseComments", licenseDescription);
if (entry.gpc != null)
record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber); // TODO
HttpEntity<WriteRecord> httpEntity = new HttpEntity<WriteRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class);
log.info("Created " + location);
FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
locations.add(location.toString());
} catch (HttpStatusCodeException hsce) {
log.error(hsce.getStatusCode().toString());
log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
log.error(hsce.getResponseBodyAsString());
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
}
}
@@ -175,7 +205,8 @@ public class MamMigrationSelective implements CommandLineRunner {
headers.add("Accept", "application/hal+json");
headers.add("Content-Type", "application/json");
headers.add("Authorization", "Basic " + adamAuth);
headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
headers.add("set-immediateSearchIndexUpdate", "true"); // see
// https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
HttpEntity<String> entity = new HttpEntity<String>("", headers);
Token token = restTemplate.exchange(adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody();
@@ -190,21 +221,16 @@ public class MamMigrationSelective implements CommandLineRunner {
}
private void invalidateAdamToken() {
ResponseEntity<String> response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE,
adamHeadersEntity, String.class);
ResponseEntity<String> response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
log.info("Logged out from adam API: " + response.toString());
}
/*
* // Quoted "Z" to indicate UTC, no timezone offset private static
* SimpleDateFormat outputDateFormat = new
* SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat
* inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
* // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new
* SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
*
* public static String toIso8601(String isoDate) { try { return
* outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
* (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return
* null; }
* public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
* (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; }
*/
public static void main(String args[]) {
AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java
+42 -49
View File
@@ -1,28 +1,17 @@
package com.bayer.edam.migration.mam;
import java.lang.reflect.Field;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
/**
* Representation of /rest/migration/entry
*/
public class Entry {
private static final Logger log = LoggerFactory.getLogger(Entry.class);
public static class Type {
public Long id;
public Map<String, String> name;
public Type() {
}
}
public static class Copyright {
@@ -43,8 +32,17 @@ public class Entry {
public Boolean regionalLicense;
public Boolean personalLicense;
public Copyright() {
}
}
public static class Gpc {
public Long id;
public Map<String, String> owner;
public Map<String, String> organization;
public String gpcState;
public String gpcNumber;
public Date expirationDate;
public String lastNotification;
public Integer gpcStateId;
}
public static class Download {
@@ -58,15 +56,13 @@ public class Entry {
public Map<String, String> description;
public String fileName;
public String fileNameOnDisc;
public Download() {
}
}
public Long id;
public String href;
public Long fileSize;
public String fileName;
public String internalKey;
public Date creationDate;
public Date uploadDate;
public Map<String, String> name;
@@ -77,14 +73,14 @@ public class Entry {
public Map<String, String> uploader;
public Map<String, String> indexer;
public Double ratingValue;
public Map<String, String> release;
public Map<String, String> release; // ??
public String brandingNet;
public Type mediaType;
public Copyright copyright;
public String master;
public Map<String, String> project;
public Type imageType;
// public String gpc;
public Gpc gpc;
public String mediaAssistantId;
public Map<String, String> metaData;
public List<String> categories;
@@ -93,42 +89,39 @@ public class Entry {
public List<Download> downloadList;
public List<Download> downloadListPublicFolder;
public Entry() {
}
@Override
public String toString() {
return "Entry{" + "id=" + id + ", name=" + name.get("en") + ", creationDate=" + creationDate + ", user="
+ user.get("id") + ", copyright.organization.id=" + copyright.organization.get("id") + '}';
}
public String toFullString() {
StringBuilder result = new StringBuilder();
String newLine = System.getProperty("line.separator");
result.append(this.getClass().getName());
result.append(" Object {");
result.append(newLine);
// determine fields declared in this class only (no fields of superclass)
Field[] fields = this.getClass().getDeclaredFields();
// print field names paired with their values
for (Field field : fields) {
result.append(" ");
try {
result.append(field.getName());
result.append(": ");
// requires access to private field:
result.append(field.get(this));
} catch (IllegalAccessException ex) {
log.error(ex.toString());
}
result.append(newLine);
}
result.append("}");
return result.toString();
}
// public String toFullString() {
// StringBuilder result = new StringBuilder();
// String newLine = System.getProperty("line.separator");
//
// result.append(this.getClass().getName());
// result.append(" Object {");
// result.append(newLine);
//
// // determine fields declared in this class only (no fields of superclass)
// Field[] fields = this.getClass().getDeclaredFields();
//
// // print field names paired with their values
// for (Field field : fields) {
// result.append(" ");
// try {
// result.append(field.getName());
// result.append(": ");
// // requires access to private field:
// result.append(field.get(this));
// } catch (IllegalAccessException ex) {
// log.error(ex.toString());
// }
// result.append(newLine);
// }
// result.append("}");
//
// return result.toString();
// }
}