diff --git a/AH MAM Migration/resources/application.properties b/AH MAM Migration/resources/application.properties index a18ad22..c70dc5a 100644 --- a/AH MAM Migration/resources/application.properties +++ b/AH MAM Migration/resources/application.properties @@ -2,4 +2,4 @@ logging.level.com.bayer=DEBUG #logging.level.com.bayer.edam.migration.interceptors=INFO logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n -logging.file=application_2017-11-27_PROD_investigation.log +logging.file=application_2017-11-28_PROD_search.log diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 267ec11..f1d4c9b 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -5,6 +5,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; @@ -47,6 +48,11 @@ import com.bayer.edam.migration.adam.Campaign; import com.bayer.edam.migration.adam.Classification; import com.bayer.edam.migration.adam.ClassificationList; import com.bayer.edam.migration.adam.EntityId; +import com.bayer.edam.migration.adam.FileList; +import com.bayer.edam.migration.adam.FileVersions; +import com.bayer.edam.migration.adam.FileVersions.FileVersion; +import com.bayer.edam.migration.adam.ReadFile; +import com.bayer.edam.migration.adam.SearchRecordResult; import com.bayer.edam.migration.adam.Token; import com.bayer.edam.migration.adam.UploadClassification; import com.bayer.edam.migration.adam.UploadFileRecord; @@ -61,6 +67,7 @@ import com.bayer.edam.migration.mam.Entry.Download; import com.fasterxml.jackson.databind.ObjectMapper; import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.exception.ZipException; import net.lingala.zip4j.model.FileHeader; @SpringBootApplication @@ -85,7 +92,7 @@ public class MamMigrationSelective implements CommandLineRunner { private String adamAuth; RestTemplate restTemplate, fileTransferTemplate; - HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders; + HttpHeaders adamHeaders, adamUploadHeaders, adamSearchHeaders, mamHeaders; HttpEntity adamHeadersEntity; private long overallUploadSize = 0; @@ -550,11 +557,16 @@ public class MamMigrationSelective implements CommandLineRunner { // Create record HttpEntity httpEntity = new HttpEntity(record, adamHeaders); try { - URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); - log.info("RESULT Entry " + entry.id + " migrated to record " + location); - - uploadFilesForEntry(entry, location); - log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); + // XXX +// URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class); +// log.info("RESULT Entry " + entry.id + " migrated to record " + location); +// +// uploadFilesForEntry(entry, location); +// log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); + + searchForRecord(entry, entryJson); + + } catch (HttpClientErrorException hcee) { log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString()); @@ -565,7 +577,133 @@ public class MamMigrationSelective implements CommandLineRunner { } + /** + * This method is used to check records that have been created after the log file has been lost :-/ + * @throws URISyntaxException + * @throws ZipException + * @throws IOException + */ + private void searchForRecord(Entry entry, String entryJson) throws URISyntaxException, IOException, ZipException { + adamSearchHeaders.set("Filter", "classification.name=Migration-MAM-KW47 " + entry.id); + log.debug("Searching for: classification.name=Migration-MAM-KW47 " + entry.id); + + // search the already existing record + SearchRecordResult searchResult = restTemplate.exchange(Definitions.adamHost + "/records", HttpMethod.GET, new HttpEntity("", adamSearchHeaders), SearchRecordResult.class).getBody(); + + if (searchResult.totalCount != 1) { + log.error("Search did not find exact match, totalCount = " + searchResult.totalCount); + return; + } + + URI location = new URI("/record/" + searchResult.items.get(0).id); + log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson); + + // Check files + FileList fileList = restTemplate.exchange(Definitions.adamHost + location.toString() + "/files", HttpMethod.GET, adamHeadersEntity, FileList.class).getBody(); + ArrayList attachedFiles = new ArrayList(); + + for (ReadFile file : fileList.items) { + FileVersions fileVersions = restTemplate.exchange(Definitions.adamHost + "/file/" + file.id + "/fileversions", HttpMethod.GET, adamHeadersEntity, FileVersions.class).getBody(); + attachedFiles.add(fileVersions); + } + + ArrayList expectedFiles = getUploadsForEntry(entry, location); + + if (expectedFiles.size() != attachedFiles.size()) { + log.error("Number of attached files does not match: attached=" + attachedFiles.size() + " expected="+expectedFiles.size()); + } + + for (File expectedFile : expectedFiles) { + int i = 0; + FileVersion fileVersion = null; + while (i < attachedFiles.size()) { + fileVersion = attachedFiles.get(i).items.get(0); + if (fileVersion.fileName.equals(expectedFile.getName())) + break; + i++; + } + + if (fileVersion.fileName.equals(expectedFile.getName())) { + log.info("Found expected file for " + location.toString() + ": " + fileVersion.fileName + ", " + fileVersion.fileSize); + } + else { + log.error("Missing file for " + location.toString() + ": " + expectedFile.getAbsolutePath()); + } + } + + + } + private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception { + + ArrayList filesForUpload = getUploadsForEntry(entry, recordLocation); + boolean isMaster = true; + + for (File file : filesForUpload) { + if (file.exists()) { + if (file.length() < 4000000000L) { + log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length()); + + MultiValueMap parameters = new LinkedMultiValueMap(); + parameters.add("file", new FileSystemResource(file)); + + try { + UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST, + new HttpEntity>(parameters, adamUploadHeaders), UploadResult.class).getBody(); + + overallUploadSize += file.length(); + log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far"); + + if (result.token != null) { + log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id); + UploadReference reference = new UploadReference(); + reference.targetRecord = recordLocation; + reference.isMaster = isMaster; + reference.token = result.token; + reference.cachePath = file.getAbsolutePath(); + openUploads.add(reference); + } + else if (result.uri != null) { + log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id); + UploadReference reference = new UploadReference(); + reference.targetRecord = recordLocation; + reference.isMaster = isMaster; + reference.uri = result.uri; + reference.cachePath = file.getAbsolutePath(); + openUploads.add(reference); + } + else { + log.error("No token/URI after upload!"); + } + + } catch (HttpClientErrorException hcee) { + log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation + + " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString()); + } catch (RestClientException rce) { + log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation + + " from entry " + entry.id + ": " + rce.getMessage()); + rce.printStackTrace(); + } + + } + else { + // TODO + log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation); + } + } + else { + log.error("Could not find file on disk " + file.getAbsolutePath()); + } + + isMaster = false; + } + + // attach uploaded files to records where possible + attachOpenUploads(); + + } + + private ArrayList getUploadsForEntry(Entry entry, URI recordLocation) throws IOException, ZipException { HashSet hrefsForUpload = new HashSet(); ArrayList filesForUpload = new ArrayList(); @@ -683,70 +821,7 @@ public class MamMigrationSelective implements CommandLineRunner { log.error("File " + download.fileName + " not available"); } } - - boolean isMaster = true; - for (File file : filesForUpload) { - if (file.exists()) { - if (file.length() < 4000000000L) { - log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length()); - - MultiValueMap parameters = new LinkedMultiValueMap(); - parameters.add("file", new FileSystemResource(file)); - - try { - UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST, - new HttpEntity>(parameters, adamUploadHeaders), UploadResult.class).getBody(); - - overallUploadSize += file.length(); - log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far"); - - if (result.token != null) { - log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id); - UploadReference reference = new UploadReference(); - reference.targetRecord = recordLocation; - reference.isMaster = isMaster; - reference.token = result.token; - reference.cachePath = file.getAbsolutePath(); - openUploads.add(reference); - } - else if (result.uri != null) { - log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id); - UploadReference reference = new UploadReference(); - reference.targetRecord = recordLocation; - reference.isMaster = isMaster; - reference.uri = result.uri; - reference.cachePath = file.getAbsolutePath(); - openUploads.add(reference); - } - else { - log.error("No token/URI after upload!"); - } - - } catch (HttpClientErrorException hcee) { - log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation - + " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString()); - } catch (RestClientException rce) { - log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation - + " from entry " + entry.id + ": " + rce.getMessage()); - rce.printStackTrace(); - } - - } - else { - // TODO - log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation); - } - } - else { - log.error("Could not find file on disk " + file.getAbsolutePath()); - } - - isMaster = false; - } - - // attach uploaded files to records where possible - attachOpenUploads(); - + return filesForUpload; } private void attachOpenUploads() { @@ -870,6 +945,10 @@ public class MamMigrationSelective implements CommandLineRunner { adamUploadHeaders = createAdamHeaders(); adamUploadHeaders.set("Content-Type", "multipart/form-data"); adamUploadHeaders.set("Accept", "*/*"); + adamSearchHeaders = createAdamHeaders(); + adamSearchHeaders.set("sort", "createdon"); + adamSearchHeaders.set("page", "1"); + adamSearchHeaders.set("pageSize", "20"); mamHeaders = createMamHeaders(); lastLogin = System.currentTimeMillis(); @@ -913,6 +992,13 @@ public class MamMigrationSelective implements CommandLineRunner { private void invalidateAdamToken() { ResponseEntity response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class); log.info("Logged out from adam API: " + response.toString()); + + response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity("", adamUploadHeaders), String.class); + log.info("Invalidated upload token: " + response.toString()); + + response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity("", adamSearchHeaders), String.class); + log.info("Invalidated search token: " + response.toString()); + } /*