tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Search for lost information

Browse Source
This commit is contained in:
GEXCE
2017-11-28 11:38:19 +01:00
parent c57203df80
commit 846c16df8e
2 changed files with 157 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/resources/application.properties
+1 -1
View File
@@ -2,4 +2,4 @@
logging.level.com.bayer=DEBUG
#logging.level.com.bayer.edam.migration.interceptors=INFO
logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
logging.file=application_2017-11-27_PROD_investigation.log
logging.file=application_2017-11-28_PROD_search.log
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+155 -69
View File
@@ -5,6 +5,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
@@ -47,6 +48,11 @@ import com.bayer.edam.migration.adam.Campaign;
import com.bayer.edam.migration.adam.Classification;
import com.bayer.edam.migration.adam.ClassificationList;
import com.bayer.edam.migration.adam.EntityId;
import com.bayer.edam.migration.adam.FileList;
import com.bayer.edam.migration.adam.FileVersions;
import com.bayer.edam.migration.adam.FileVersions.FileVersion;
import com.bayer.edam.migration.adam.ReadFile;
import com.bayer.edam.migration.adam.SearchRecordResult;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadFileRecord;
@@ -61,6 +67,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import net.lingala.zip4j.model.FileHeader;
@SpringBootApplication
@@ -85,7 +92,7 @@ public class MamMigrationSelective implements CommandLineRunner {
private String adamAuth;
RestTemplate restTemplate, fileTransferTemplate;
HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
HttpHeaders adamHeaders, adamUploadHeaders, adamSearchHeaders, mamHeaders;
HttpEntity<String> adamHeadersEntity;
private long overallUploadSize = 0;
@@ -550,11 +557,16 @@ public class MamMigrationSelective implements CommandLineRunner {
// Create record
HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
log.info("RESULT Entry " + entry.id + " migrated to record " + location);
// XXX
// URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
// log.info("RESULT Entry " + entry.id + " migrated to record " + location);
//
// uploadFilesForEntry(entry, location);
// log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
searchForRecord(entry, entryJson);
uploadFilesForEntry(entry, location);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
} catch (HttpClientErrorException hcee) {
log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + hcee.getMessage() + ", "
+ hcee.getResponseBodyAsString());
@@ -565,7 +577,133 @@ public class MamMigrationSelective implements CommandLineRunner {
}
/**
* This method is used to check records that have been created after the log file has been lost :-/
* @throws URISyntaxException
* @throws ZipException
* @throws IOException
*/
private void searchForRecord(Entry entry, String entryJson) throws URISyntaxException, IOException, ZipException {
adamSearchHeaders.set("Filter", "classification.name=Migration-MAM-KW47 " + entry.id);
log.debug("Searching for: classification.name=Migration-MAM-KW47 " + entry.id);
// search the already existing record
SearchRecordResult searchResult = restTemplate.exchange(Definitions.adamHost + "/records", HttpMethod.GET, new HttpEntity<String>("", adamSearchHeaders), SearchRecordResult.class).getBody();
if (searchResult.totalCount != 1) {
log.error("Search did not find exact match, totalCount = " + searchResult.totalCount);
return;
}
URI location = new URI("/record/" + searchResult.items.get(0).id);
log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
// Check files
FileList fileList = restTemplate.exchange(Definitions.adamHost + location.toString() + "/files", HttpMethod.GET, adamHeadersEntity, FileList.class).getBody();
ArrayList<FileVersions> attachedFiles = new ArrayList<FileVersions>();
for (ReadFile file : fileList.items) {
FileVersions fileVersions = restTemplate.exchange(Definitions.adamHost + "/file/" + file.id + "/fileversions", HttpMethod.GET, adamHeadersEntity, FileVersions.class).getBody();
attachedFiles.add(fileVersions);
}
ArrayList<File> expectedFiles = getUploadsForEntry(entry, location);
if (expectedFiles.size() != attachedFiles.size()) {
log.error("Number of attached files does not match: attached=" + attachedFiles.size() + " expected="+expectedFiles.size());
}
for (File expectedFile : expectedFiles) {
int i = 0;
FileVersion fileVersion = null;
while (i < attachedFiles.size()) {
fileVersion = attachedFiles.get(i).items.get(0);
if (fileVersion.fileName.equals(expectedFile.getName()))
break;
i++;
}
if (fileVersion.fileName.equals(expectedFile.getName())) {
log.info("Found expected file for " + location.toString() + ": " + fileVersion.fileName + ", " + fileVersion.fileSize);
}
else {
log.error("Missing file for " + location.toString() + ": " + expectedFile.getAbsolutePath());
}
}
}
private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
ArrayList<File> filesForUpload = getUploadsForEntry(entry, recordLocation);
boolean isMaster = true;
for (File file : filesForUpload) {
if (file.exists()) {
if (file.length() < 4000000000L) {
log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(file));
try {
UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
overallUploadSize += file.length();
log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
if (result.token != null) {
log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
reference.token = result.token;
reference.cachePath = file.getAbsolutePath();
openUploads.add(reference);
}
else if (result.uri != null) {
log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
reference.uri = result.uri;
reference.cachePath = file.getAbsolutePath();
openUploads.add(reference);
}
else {
log.error("No token/URI after upload!");
}
} catch (HttpClientErrorException hcee) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ": " + rce.getMessage());
rce.printStackTrace();
}
}
else {
// TODO
log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation);
}
}
else {
log.error("Could not find file on disk " + file.getAbsolutePath());
}
isMaster = false;
}
// attach uploaded files to records where possible
attachOpenUploads();
}
private ArrayList<File> getUploadsForEntry(Entry entry, URI recordLocation) throws IOException, ZipException {
HashSet<String> hrefsForUpload = new HashSet<String>();
ArrayList<File> filesForUpload = new ArrayList<File>();
@@ -683,70 +821,7 @@ public class MamMigrationSelective implements CommandLineRunner {
log.error("File " + download.fileName + " not available");
}
}
boolean isMaster = true;
for (File file : filesForUpload) {
if (file.exists()) {
if (file.length() < 4000000000L) {
log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
parameters.add("file", new FileSystemResource(file));
try {
UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
overallUploadSize += file.length();
log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
if (result.token != null) {
log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
reference.token = result.token;
reference.cachePath = file.getAbsolutePath();
openUploads.add(reference);
}
else if (result.uri != null) {
log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
UploadReference reference = new UploadReference();
reference.targetRecord = recordLocation;
reference.isMaster = isMaster;
reference.uri = result.uri;
reference.cachePath = file.getAbsolutePath();
openUploads.add(reference);
}
else {
log.error("No token/URI after upload!");
}
} catch (HttpClientErrorException hcee) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
} catch (RestClientException rce) {
log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+ " from entry " + entry.id + ": " + rce.getMessage());
rce.printStackTrace();
}
}
else {
// TODO
log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation);
}
}
else {
log.error("Could not find file on disk " + file.getAbsolutePath());
}
isMaster = false;
}
// attach uploaded files to records where possible
attachOpenUploads();
return filesForUpload;
}
private void attachOpenUploads() {
@@ -870,6 +945,10 @@ public class MamMigrationSelective implements CommandLineRunner {
adamUploadHeaders = createAdamHeaders();
adamUploadHeaders.set("Content-Type", "multipart/form-data");
adamUploadHeaders.set("Accept", "*/*");
adamSearchHeaders = createAdamHeaders();
adamSearchHeaders.set("sort", "createdon");
adamSearchHeaders.set("page", "1");
adamSearchHeaders.set("pageSize", "20");
mamHeaders = createMamHeaders();
lastLogin = System.currentTimeMillis();
@@ -913,6 +992,13 @@ public class MamMigrationSelective implements CommandLineRunner {
private void invalidateAdamToken() {
ResponseEntity<String> response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
log.info("Logged out from adam API: " + response.toString());
response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity<String>("", adamUploadHeaders), String.class);
log.info("Invalidated upload token: " + response.toString());
response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity<String>("", adamSearchHeaders), String.class);
log.info("Invalidated search token: " + response.toString());
}
/*