From a88d58752d8bd1c776e2eb31a546cba1ebc5bca3 Mon Sep 17 00:00:00 2001 From: gexce Date: Wed, 20 Sep 2017 16:21:06 +0200 Subject: [PATCH] EntryDownloader --- AH MAM Migration/questions.txt | 48 ++++++++++-- .../bayer/edam/migration/EntryDownloader.java | 74 +++++++++++++++++++ .../edam/migration/MamMigrationSelective.java | 9 +-- .../com/bayer/edam/migration/mam/Entry.java | 5 +- .../bayer/edam/migration/mam/Entrylist.java | 17 +++++ 5 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java create mode 100644 AH MAM Migration/src/com/bayer/edam/migration/mam/Entrylist.java diff --git a/AH MAM Migration/questions.txt b/AH MAM Migration/questions.txt index 1c41c23..366c9d1 100644 --- a/AH MAM Migration/questions.txt +++ b/AH MAM Migration/questions.txt @@ -1,6 +1,6 @@ - English only? - CreationDate ist read-only -- Mapping Media Type +- Mapping Media Type --> bringt Martin mit - Wer soll als byr_ah_projectLead gesetzt werden? Brand Comm representative für beide Felder? - Soll muss der MAM-Uploader irgendwo gespeichert werden? - "Bayer Animal Health GmbH" immer als License Holder OK? @@ -8,14 +8,25 @@ - MAM-Status "ARCHIVED" --> eDAM.Archived? - Was ist mit Status != ACCEPTED or ARCHIVED? - Regelung Cutover: Kein Upload mehr ab KW42 (16.10.) -- Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763) -- Beispiel für ausgefüllte GPC-Daten + +License: +- Wenn License Holder nicht gesetzt, trotzdem BAH GmbH? (bspw. 00070763) --> mandatory +- release.forInternet/forIntranet --> License comments +- GPC owner/e-mail? + +GPC: +- Beispiel für ausgefüllte GPC-Daten/Wird GPC-Block verwendet? - was ist mit regionalLicenseComment/personalLicenseComment? -- Wird GPC-Block verwendet? -- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = Retire Date? +- Ist "Date of Issue" Release Date? Ist gpc.expirationDate = LMR Expiration Date? - GPC No. = LMR ID? +- Timeline + - Fertigstellung Migrations-Programm + - Download/Upload Metadaten + - Download/Upload Dateien + + CREATE TABLE Record ( ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, Location VARCHAR NOT NULL, @@ -30,3 +41,30 @@ SELECT * FROM Record; CALL CSVWRITE ('C:/Temp/Migration AH/entries.csv', 'SELECT * FROM Record', 'charset=UTF-8'); + + +DROP ALL OBJECTS + + +CREATE TABLE Entry ( + ID INTEGER GENERATED BY DEFAULT AS IDENTITY(START WITH 0) NOT NULL PRIMARY KEY, + EntryId INTEGER NOT NULL, + EntryIdString VARCHAR NOT NULL, + MediaType INTEGER NOT NULL, + Filesize BIGINT NOT NULL, + Json VARCHAR NOT NULL) +AS SELECT + null, + EntryId, + EntryIdString, + MediaType, + Filesize, + Json +FROM CSVREAD('C:/Temp/Migration AH/migration.log', 'EntryId|~|EntryIdString|~|MediaType|~|Filesize|~|Json', 'charset=UTF-8 fieldSeparator=|~|'); + +SELECT * FROM Entry; + +CALL CSVWRITE ('C:/Temp/Migration AH/migration.csv', 'SELECT * FROM Entry', 'charset=UTF-8'); + + + diff --git a/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java new file mode 100644 index 0000000..b0fc339 --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java @@ -0,0 +1,74 @@ +package com.bayer.edam.migration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.PropertySource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.web.client.RestTemplate; + +import com.bayer.edam.migration.mam.Entry; +import com.bayer.edam.migration.mam.Entry.Download; +import com.bayer.edam.migration.mam.Entrylist; +import com.fasterxml.jackson.databind.ObjectMapper; + +@SpringBootApplication +@PropertySource("file:secret.properties") +public class EntryDownloader implements CommandLineRunner { + + private static final Logger log = LoggerFactory.getLogger(EntryDownloader.class); + + @Value("${mam.auth}") + private String mamAuth; + + + static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; + + static final String separator = "|~|"; + + + private ObjectMapper jsonMapper = new ObjectMapper(); + private RestTemplate restTemplate = new RestTemplate(); + + + @Override + public void run(String... strings) throws Exception { + HttpHeaders headers = new HttpHeaders(); + headers.add("Accept", "application/json"); + headers.add("Authorization", "Basic " + mamAuth); + headers.add("Content-Type", "application/json"); + headers.add("From", "migration@bayer.com"); + HttpEntity entity = new HttpEntity("", headers); + + int lastPage = 1; + for (int pageNum = 1; pageNum <= lastPage; pageNum++) { + Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody(); + lastPage = entrylist.lastPage; + + for (int i = 0; i < entrylist.entries.length; i++) { + String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody(); + Entry entry = jsonMapper.readValue(entryJson, Entry.class); + + long aggregatedFilesize = 0; + for (Download download : entry.downloadList) { + aggregatedFilesize += download.size; + } + for (Download download : entry.downloadListPublicFolder) { + aggregatedFilesize += download.size; + } + + log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson); + } + } + } + + public static void main(String args[]) { + SpringApplication.run(EntryDownloader.class, args); + } + +} diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 12f71c8..5dd836e 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -34,7 +34,7 @@ import com.bayer.edam.migration.adam.WriteRecord; import com.bayer.edam.migration.mam.Entry; import com.fasterxml.jackson.databind.ObjectMapper; -@SpringBootApplication +//@SpringBootApplication @PropertySource("file:secret.properties") public class MamMigrationSelective implements CommandLineRunner { @@ -60,10 +60,9 @@ public class MamMigrationSelective implements CommandLineRunner { HttpHeaders adamHeaders, mamHeaders; HttpEntity adamHeadersEntity; - private static SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); - private static SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); - - private static ObjectMapper jsonMapper = new ObjectMapper(); + private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); + private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd"); + private ObjectMapper jsonMapper = new ObjectMapper(); private List locations = new ArrayList(); diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java index 24e813a..753b816 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entry.java @@ -4,9 +4,12 @@ import java.util.Date; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Representation of /rest/migration/entry */ +@JsonIgnoreProperties(ignoreUnknown = true) public class Entry { public static class Type { @@ -37,7 +40,6 @@ public class Entry { public static class Gpc { public Long id; public Map owner; - public Map organization; public String gpcState; public String gpcNumber; public Date expirationDate; @@ -45,6 +47,7 @@ public class Entry { public Integer gpcStateId; } + @JsonIgnoreProperties(ignoreUnknown = true) public static class Download { public Long id; public String href; diff --git a/AH MAM Migration/src/com/bayer/edam/migration/mam/Entrylist.java b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entrylist.java new file mode 100644 index 0000000..2a9abdc --- /dev/null +++ b/AH MAM Migration/src/com/bayer/edam/migration/mam/Entrylist.java @@ -0,0 +1,17 @@ +package com.bayer.edam.migration.mam; + +public class Entrylist { + + public static class EntryOverview { + public String mediaAssistantId; + public int entryStateId; + public int mediaTypeId; + } + + public int totalHits; + public int currentPage; + public int lastPage; + public int pageSize; + public EntryOverview[] entries; + +}