tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

getting closer

Browse Source
This commit is contained in:
gexce
2017-11-15 10:55:32 +01:00
parent 8cc74753bf
commit 78f40e7625
8 changed files with 907 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+18 -15
View File
@@ -62,7 +62,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 71725, 71263, 71048, 70778, 70746, 70745, 70440, 70361, 70340, 70288, 70161, 70142, 63924, 63201, 57641, 56942 ));
private List<Integer> entrySelection = new ArrayList<Integer>(Arrays.asList( 46061/*71725, 71263, 71048, 70778, 70746, 70745, 70440, 70361, 70340, 70288, 70161, 70142, 63924, 63201, 57641, 56942*/ ));
static final String fileCache = "E:\\";
@@ -70,8 +70,8 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
// XXX
// static final String entryListPath = "resources/entrylist_2017-10-25.txt";
static final String entryListPath = "resources/lottery.txt";
// static final String entryListPath = "resources/entrylist_2017-11-09.txt";
static final String entryListPath = "resources/lottery-min.txt";
@Value("${mam.auth}")
private String mamAuth;
@@ -100,7 +100,7 @@ public class MamMigrationSelective implements CommandLineRunner {
public void run(String... strings) throws Exception {
// XXX
readEntryIDsFromFile();
// readEntryIDsFromFile();
log.info("Read categories");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
@@ -132,17 +132,18 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
int entryCounter = 1;
HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
for (int id : entrySelection) {
// XXX
// // if needed start from a certain ID
// if (id == 70440)
// if (id > 44741)
// continue;
log.info("GET entry " + id);
log.info("GET entry " + id + " (" + entryCounter++ + ")");
Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
@@ -171,7 +172,7 @@ public class MamMigrationSelective implements CommandLineRunner {
while (openUploads.size() > 0) {
log.debug("Sleeping...");
TimeUnit.SECONDS.sleep(10); // XXX
TimeUnit.SECONDS.sleep(10);
log.debug("Checking back on open uploads");
attachOpenUploads();
@@ -184,6 +185,7 @@ public class MamMigrationSelective implements CommandLineRunner {
UploadRecord record = new UploadRecord();
String entryJson = jsonMapper.writeValueAsString(entry);
List<String> keywords = new ArrayList<String>();
HashSet<String> keywordSet = new HashSet<String>();
HashSet<String> campaignGuids = new HashSet<String>();
// XXX
@@ -379,8 +381,10 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
else if (n.action.equals("keyword")) {
log.debug("Adding keyword " + n.name.trim());
keywords.add(n.name.trim());
if (keywordSet.add(n.name.trim().toLowerCase())) {
log.debug("Adding keyword " + n.name.trim());
keywords.add(n.name.trim());
}
}
}
@@ -455,7 +459,6 @@ public class MamMigrationSelective implements CommandLineRunner {
// Keywords
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
HashSet<String> keywordSet = new HashSet<String>();
for (String s : mamKeywords) {
if (keywordSet.add(s.trim().toLowerCase())) {
@@ -501,7 +504,7 @@ public class MamMigrationSelective implements CommandLineRunner {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
uploadFilesForEntry(entry, location);
// uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
}
@@ -603,7 +606,7 @@ public class MamMigrationSelective implements CommandLineRunner {
openUploads.add(reference);
}
else if (result.uri != null) {
log.debug("Storing upload reference " + result.uri + "(" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
log.debug("Storing upload reference " + result.uri + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
@@ -617,7 +620,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
catch (ResourceAccessException rae) {
log.error("Could not upload " + file.getAbsolutePath() + "(" + isMaster + ") for " + recordLocation + " from entry " + entry.id + ": " + rae.getMessage());
log.error("Could not upload " + file.getAbsolutePath() + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id + ": " + rae.getMessage());
rae.printStackTrace();
}
@@ -647,7 +650,7 @@ public class MamMigrationSelective implements CommandLineRunner {
UploadFileRecord fileRecord = new UploadFileRecord();
fileRecord.addFile(reference.token, reference.isMaster);
log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord);
log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord + " via " + reference.token + " from " + reference.uri);
HttpEntity<UploadFileRecord> httpEntity = new HttpEntity<UploadFileRecord>(fileRecord, adamHeaders);
try {
@@ -668,7 +671,6 @@ public class MamMigrationSelective implements CommandLineRunner {
if (result.token != null) {
log.debug("Storing upload token " + result.token);
reference.uri = null;
reference.token = result.token;
}
else if (result.status != null) {
@@ -681,6 +683,7 @@ public class MamMigrationSelective implements CommandLineRunner {
}
openUploads.removeAll(processedReferences);
log.debug(openUploads.size() + " open uploads remaining");
}
private File getFileForEntry(Download download, Long id) throws IOException {