tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Renew login

Browse Source
This commit is contained in:
gexce
2017-11-21 09:59:41 +01:00
parent b790b090e4
commit a12c41c776
4 changed files with 425 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+102 -55
View File
@@ -9,6 +9,7 @@ import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -37,6 +38,7 @@ import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
@@ -85,7 +87,13 @@ public class MamMigrationSelective implements CommandLineRunner {
RestTemplate restTemplate, fileTransferTemplate;
HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
HttpEntity<String> adamHeadersEntity;
private long overallUploadSize = 0;
private int entryCounter = 0;
private int skippedCounter = 0;
private long lastLogin = 0;
private static DecimalFormat decimalFormat = new DecimalFormat("#,##0.#");
private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private ObjectMapper jsonMapper = new ObjectMapper();
@@ -111,12 +119,11 @@ public class MamMigrationSelective implements CommandLineRunner {
// XXX ----------------------------------------------------------------
// Option 1: Process all available entries
// entrySelection = new ArrayList<Long>(entryMap.keySet());
entrySelection = new ArrayList<Long>(entryMap.keySet());
// Option 2: Process a fixed set of entry IDs
// entrySelection = Arrays.asList(71725L, 71537L, 71263L, 71048L, 70778L, 70746L, 70745L,
// 70440L, 70361L, 70340L, 70288L, 70161L, 70142L, 63924L, 63201L, 57641L, 56942L);
entrySelection = Arrays.asList(34197L);
// Option 3: Read entry IDs from file
// entrySelection = new ArrayList<Long>();
@@ -126,7 +133,8 @@ public class MamMigrationSelective implements CommandLineRunner {
// }
// --------------------------------------------------------------------
Collections.sort(entrySelection);
Collections.reverse(entrySelection);
log.info("Read categories");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
@@ -139,13 +147,8 @@ public class MamMigrationSelective implements CommandLineRunner {
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
interceptors.add(new LoggingRequestInterceptor());
restTemplate.setInterceptors(interceptors);
adamHeaders = createAdamHeaders();
adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
adamUploadHeaders = createAdamHeaders();
adamUploadHeaders.set("Content-Type", "multipart/form-data");
adamUploadHeaders.set("Accept", "*/*");
mamHeaders = createMamHeaders();
recreateHeaders();
log.info("Read campaigns");
campaignYears = restTemplate.exchange(Definitions.adamHost + "/classification/" + Definitions.campaignRoot + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
@@ -158,20 +161,22 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
int entryCounter = 1;
// HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
for (long id : entrySelection) {
// XXX
// // if needed start from a certain ID
// if (id > 44741)
// continue;
// if needed start from a certain ID
if (id > 71715)
continue;
// refresh login every 4 hours
if (System.currentTimeMillis() - lastLogin > 14400000)
recreateHeaders();
log.info("GET entry " + id + " (" + entryCounter++ + "/" + entrySelection.size() + ")");
entryCounter++;
log.info("GET entry " + id + " (" + entryCounter + "/" + entrySelection.size() + " - " + skippedCounter + ")");
// Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
Entry entry = entryMap.get(id);
@@ -181,18 +186,22 @@ public class MamMigrationSelective implements CommandLineRunner {
// check exclusion criteria
if ((entry.stateId == 5 || entry.stateId == 4) && entry.fileName == null && entry.categories.size() == 0) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + ", no file attached, no category set");
skippedCounter++;
continue;
}
else if (entry.stateId == 7) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (LOCKED_BY_ADMIN)");
skippedCounter++;
continue;
}
else if (entry.stateId == 8) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ASSIGNED_TO_INDEXATION)");
skippedCounter++;
continue;
}
else if (entry.stateId == 10) {
log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ARCHIVED)");
skippedCounter++;
continue;
}
@@ -232,6 +241,7 @@ public class MamMigrationSelective implements CommandLineRunner {
// process categories
if (entry.categories.size() == 0) {
log.info("RESULT Skipping entry " + entry.id + ": No category assigned, cannot determine business unit and brand");
skippedCounter++;
return;
}
@@ -246,10 +256,17 @@ public class MamMigrationSelective implements CommandLineRunner {
List<Node> path = new LinkedList<Node>();
Node tmpNode = node;
if (tmpNode.id == CategoryTree.ROOT_NODE_ID) {
log.info("RESULT Skipping entry " + entry.id + ": Node in invisible root of category tree (" + node.toString() + ")");
skippedCounter++;
return;
}
String currentBrand = null;
while (tmpNode.id != CategoryTree.ROOT_NODE_ID) {
if (tmpNode.id == 0) {
log.info("RESULT Skipping entry " + entry.id + ": Node outside of category tree (" + node.toString() + ")");
skippedCounter++;
return;
}
@@ -260,6 +277,7 @@ public class MamMigrationSelective implements CommandLineRunner {
for (Node n : path) {
if (n.action.equals("not migrated")) {
log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")");
skippedCounter++;
return;
}
@@ -531,25 +549,16 @@ public class MamMigrationSelective implements CommandLineRunner {
// Create record
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
// URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
ResponseEntity<String> response = restTemplate.exchange(Definitions.adamHost + "/records", HttpMethod.POST, httpEntity, String.class);
// XXX
if (response.getStatusCodeValue() == 201) {
UploadResult result = jsonMapper.readValue(response.getBody(), UploadResult.class);
URI location = new URI("/record/" + result.id);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
}
else {
log.error("RESULT Could not create record for entry " + entry.id + ": " + );
}
}
catch (RestClientException rce) {
log.error("RESULT Could not create record for entry " + entry.id + ": " + rce.getMessage());
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
} catch (HttpClientErrorException hcee) {
log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + hcee.getMessage() + ", "
+ hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + rce.getMessage());
rce.printStackTrace();
}
}
@@ -674,7 +683,6 @@ public class MamMigrationSelective implements CommandLineRunner {
}
}
// Instead of downloading from MAM, take the pre-downloaded files from the harddrive
boolean isMaster = true;
for (File file : filesForUpload) {
if (file.exists()) {
@@ -688,8 +696,11 @@ public class MamMigrationSelective implements CommandLineRunner {
UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
overallUploadSize += file.length();
log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
if (result.token != null) {
log.debug("Storing upload token " + result.token + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
@@ -698,7 +709,7 @@ public class MamMigrationSelective implements CommandLineRunner {
openUploads.add(reference);
}
else if (result.uri != null) {
log.debug("Storing upload reference " + result.uri + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id);
log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
@@ -710,9 +721,12 @@ public class MamMigrationSelective implements CommandLineRunner {
log.error("No token/URI after upload!");
}
}
catch (RestClientException rce) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + " (" + isMaster + ") for " + recordLocation + " from entry " + entry.id + ": " + rce.getMessage());
} catch (HttpClientErrorException hcee) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ": " + rce.getMessage());
rce.printStackTrace();
}
@@ -742,15 +756,19 @@ public class MamMigrationSelective implements CommandLineRunner {
UploadFileRecord fileRecord = new UploadFileRecord();
fileRecord.addFile(reference.token, reference.isMaster);
log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord + " via " + reference.token + " from " + reference.uri);
log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord + " via " + reference.token);
HttpEntity<UploadFileRecord> httpEntity = new HttpEntity<UploadFileRecord>(fileRecord, adamHeaders);
try {
restTemplate.exchange(Definitions.adamHost + reference.targetRecord, HttpMethod.PUT, httpEntity, Void.class);
log.debug("ATTACH RESULT Attached " + reference.cachePath + " (" + reference.isMaster + ") to " + reference.targetRecord + " via " + reference.token);
}
catch (RestClientException rce) {
log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath + " (" + reference.isMaster + ") for " + reference.targetRecord + ": " + rce.getMessage());
log.info("ATTACH RESULT Attached " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord);
} catch (HttpClientErrorException hcee) {
log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath
+ (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord + ", Exception: "
+ hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath
+ (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord + ": " + rce.getMessage());
rce.printStackTrace();
}
@@ -759,28 +777,35 @@ public class MamMigrationSelective implements CommandLineRunner {
else {
log.debug("Checking status of " + reference.cachePath + " for " + reference.targetRecord + " at " + reference.uri);
try {
UploadResult result = restTemplate.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class).getBody();
UploadResult result = restTemplate
.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class)
.getBody();
log.debug("Result: " + result);
if (result.token != null) {
log.debug("Storing upload token " + result.token);
reference.token = result.token;
}
else if (result.status != null) {
} else if (result.status != null) {
if (result.status.equals("Error")) {
log.error("System could not process file " + reference.cachePath + " for " + reference.targetRecord);
processedReferences.add(reference);
}
}
}
catch (RestClientException rce) {
log.error("Failed to check status of " + reference.cachePath + " for " + reference.uri + ": " + rce.getMessage());
} catch (HttpClientErrorException hcee) {
log.error("Failed to check status of " + reference.cachePath + " for " + reference.targetRecord + ", Exception: "
+ hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("Failed to check status of " + reference.cachePath + " for " + reference.targetRecord + ": " + rce.getMessage());
}
}
}
openUploads.removeAll(processedReferences);
log.debug(openUploads.size() + " open uploads remaining");
log.info(openUploads.size() + " open uploads remaining");
for (UploadReference reference : openUploads) {
log.debug(" " + reference.uri + " for " + reference.targetRecord + (reference.isMaster ? " (master)" : "") + " as " + reference.cachePath);
}
}
private File getFileForEntry(Download download, Long id) throws IOException {
@@ -835,8 +860,21 @@ public class MamMigrationSelective implements CommandLineRunner {
return entryMap;
}
private void recreateHeaders() {
adamHeaders = createAdamHeaders();
adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
adamUploadHeaders = createAdamHeaders();
adamUploadHeaders.set("Content-Type", "multipart/form-data");
adamUploadHeaders.set("Accept", "*/*");
mamHeaders = createMamHeaders();
lastLogin = System.currentTimeMillis();
}
private HttpHeaders createMamHeaders() {
log.debug("MediaAssistant login");
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/json");
headers.add("Authorization", "Basic " + mamAuth);
@@ -847,6 +885,8 @@ public class MamMigrationSelective implements CommandLineRunner {
}
private HttpHeaders createAdamHeaders() {
log.debug("Adam login");
HttpHeaders headers = new HttpHeaders();
headers.add("Registration", "ADAMBAYER");
headers.add("API-VERSION", "1");
@@ -879,6 +919,13 @@ public class MamMigrationSelective implements CommandLineRunner {
* public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
* (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; }
*/
public static String readableFileSize(long size) {
if(size <= 0) return "0";
final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" };
int digitGroups = (int) (Math.log10(size)/Math.log10(1024));
return decimalFormat.format(size/Math.pow(1024, digitGroups)) + " " + units[digitGroups];
}
public static void main(String args[]) {
SpringApplication.run(MamMigrationSelective.class, args);