Search for lost information

AH MAM Migration/resources/application.properties

@@ -2,4 +2,4 @@
 logging.level.com.bayer=DEBUG
 #logging.level.com.bayer.edam.migration.interceptors=INFO
 logging.pattern.console=%d{yyyy-MM-dd HH:mm:ss} %-5level- %msg%n
-logging.file=application_2017-11-27_PROD_investigation.log
+logging.file=application_2017-11-28_PROD_search.log

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -5,6 +5,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
@@ -47,6 +48,11 @@ import com.bayer.edam.migration.adam.Campaign;
 import com.bayer.edam.migration.adam.Classification;
 import com.bayer.edam.migration.adam.ClassificationList;
 import com.bayer.edam.migration.adam.EntityId;
+import com.bayer.edam.migration.adam.FileList;
+import com.bayer.edam.migration.adam.FileVersions;
+import com.bayer.edam.migration.adam.FileVersions.FileVersion;
+import com.bayer.edam.migration.adam.ReadFile;
+import com.bayer.edam.migration.adam.SearchRecordResult;
 import com.bayer.edam.migration.adam.Token;
 import com.bayer.edam.migration.adam.UploadClassification;
 import com.bayer.edam.migration.adam.UploadFileRecord;
@@ -61,6 +67,7 @@ import com.bayer.edam.migration.mam.Entry.Download;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import net.lingala.zip4j.core.ZipFile;
+import net.lingala.zip4j.exception.ZipException;
 import net.lingala.zip4j.model.FileHeader;
 
 @SpringBootApplication
@@ -85,7 +92,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private String adamAuth;
 
 	RestTemplate restTemplate, fileTransferTemplate;
-	HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
+	HttpHeaders adamHeaders, adamUploadHeaders, adamSearchHeaders, mamHeaders;
 	HttpEntity<String> adamHeadersEntity;
 	
 	private long overallUploadSize = 0;
@@ -550,11 +557,16 @@ public class MamMigrationSelective implements CommandLineRunner {
 		// Create record
 		HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
 		try {
-			URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
+			// XXX
-			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
+//			URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
+//			log.info("RESULT Entry " + entry.id + " migrated to record " + location);
+//			
+//			uploadFilesForEntry(entry, location);
+//			log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
+			
+			searchForRecord(entry, entryJson);
+			
 			
-			uploadFilesForEntry(entry, location);
-			log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
 		} catch (HttpClientErrorException hcee) {
 			log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + hcee.getMessage() + ", "
 					+ hcee.getResponseBodyAsString());
@@ -565,7 +577,133 @@ public class MamMigrationSelective implements CommandLineRunner {
 	}
 	
 	
+	/**
+	 * This method is used to check records that have been created after the log file has been lost :-/
+	 * @throws URISyntaxException 
+	 * @throws ZipException 
+	 * @throws IOException 
+	 */
+	private void searchForRecord(Entry entry, String entryJson) throws URISyntaxException, IOException, ZipException {
+		adamSearchHeaders.set("Filter", "classification.name=Migration-MAM-KW47 " + entry.id);
+		log.debug("Searching for: classification.name=Migration-MAM-KW47 " + entry.id);
+		
+		// search the already existing record
+		SearchRecordResult searchResult = restTemplate.exchange(Definitions.adamHost + "/records", HttpMethod.GET, new HttpEntity<String>("", adamSearchHeaders), SearchRecordResult.class).getBody();
+		
+		if (searchResult.totalCount != 1) {
+			log.error("Search did not find exact match, totalCount = " + searchResult.totalCount);
+			return;
+		}
+		
+		URI location = new URI("/record/" + searchResult.items.get(0).id);
+		log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
+		
+		// Check files
+		FileList fileList = restTemplate.exchange(Definitions.adamHost + location.toString() + "/files", HttpMethod.GET, adamHeadersEntity, FileList.class).getBody();
+		ArrayList<FileVersions> attachedFiles = new ArrayList<FileVersions>();
+		
+		for (ReadFile file : fileList.items) {
+			FileVersions fileVersions = restTemplate.exchange(Definitions.adamHost + "/file/" + file.id + "/fileversions", HttpMethod.GET, adamHeadersEntity, FileVersions.class).getBody();
+			attachedFiles.add(fileVersions);
+		}
+		
+		ArrayList<File> expectedFiles = getUploadsForEntry(entry, location);
+		
+		if (expectedFiles.size() != attachedFiles.size()) {
+			log.error("Number of attached files does not match: attached=" + attachedFiles.size() + " expected="+expectedFiles.size());
+		}
+		
+		for (File expectedFile : expectedFiles) {
+			int i = 0;
+			FileVersion fileVersion = null;
+			while (i < attachedFiles.size()) {
+				fileVersion = attachedFiles.get(i).items.get(0);
+				if (fileVersion.fileName.equals(expectedFile.getName()))
+					break;
+				i++;
+			}
+			
+			if (fileVersion.fileName.equals(expectedFile.getName())) {
+				log.info("Found expected file for " + location.toString() + ": " + fileVersion.fileName + ", " + fileVersion.fileSize);
+			}
+			else {
+				log.error("Missing file for " + location.toString() + ": " + expectedFile.getAbsolutePath());
+			}
+		}
+		
+		
+	}
+
 	private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
+		
+		ArrayList<File> filesForUpload = getUploadsForEntry(entry, recordLocation);
+		boolean isMaster = true;
+		
+		for (File file : filesForUpload) {
+			if (file.exists()) {
+				if (file.length() < 4000000000L) {
+					log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
+					
+					MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
+					parameters.add("file", new FileSystemResource(file));
+					
+					try {
+						UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
+						new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
+						
+						overallUploadSize += file.length();
+						log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
+						
+						if (result.token != null) {
+							log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
+							UploadReference reference = new UploadReference();
+							reference.targetRecord = recordLocation;
+							reference.isMaster = isMaster;
+							reference.token = result.token;
+							reference.cachePath = file.getAbsolutePath();
+							openUploads.add(reference);
+						}
+						else if (result.uri != null) {
+							log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
+							UploadReference reference = new UploadReference();
+							reference.targetRecord = recordLocation;
+							reference.isMaster = isMaster;
+							reference.uri = result.uri;
+							reference.cachePath = file.getAbsolutePath();
+							openUploads.add(reference);
+						}
+						else {
+							log.error("No token/URI after upload!");
+						}
+
+					} catch (HttpClientErrorException hcee) {
+						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+								+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
+					} catch (RestClientException rce) {
+						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
+								+ " from entry " + entry.id + ": " + rce.getMessage());
+						rce.printStackTrace();
+					}
+					
+				}
+				else {
+					// TODO
+					log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation);
+				}
+			}
+			else {
+				log.error("Could not find file on disk " + file.getAbsolutePath());
+			}
+			
+			isMaster = false;
+		}
+		
+		// attach uploaded files to records where possible
+		attachOpenUploads();
+		
+	}
+
+	private ArrayList<File> getUploadsForEntry(Entry entry, URI recordLocation) throws IOException, ZipException {
 		HashSet<String> hrefsForUpload = new HashSet<String>();
 		ArrayList<File> filesForUpload = new ArrayList<File>();
 		
@@ -683,70 +821,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 					log.error("File " + download.fileName + " not available");
 			}
 		}
-		
+		return filesForUpload;
-		boolean isMaster = true;
-		for (File file : filesForUpload) {
-			if (file.exists()) {
-				if (file.length() < 4000000000L) {
-					log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
-					
-					MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
-					parameters.add("file", new FileSystemResource(file));
-					
-					try {
-						UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
-						new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
-						
-						overallUploadSize += file.length();
-						log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
-						
-						if (result.token != null) {
-							log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
-							UploadReference reference = new UploadReference();
-							reference.targetRecord = recordLocation;
-							reference.isMaster = isMaster;
-							reference.token = result.token;
-							reference.cachePath = file.getAbsolutePath();
-							openUploads.add(reference);
-						}
-						else if (result.uri != null) {
-							log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
-							UploadReference reference = new UploadReference();
-							reference.targetRecord = recordLocation;
-							reference.isMaster = isMaster;
-							reference.uri = result.uri;
-							reference.cachePath = file.getAbsolutePath();
-							openUploads.add(reference);
-						}
-						else {
-							log.error("No token/URI after upload!");
-						}
-
-					} catch (HttpClientErrorException hcee) {
-						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
-								+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
-					} catch (RestClientException rce) {
-						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
-								+ " from entry " + entry.id + ": " + rce.getMessage());
-						rce.printStackTrace();
-					}
-					
-				}
-				else {
-					// TODO
-					log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation);
-				}
-			}
-			else {
-				log.error("Could not find file on disk " + file.getAbsolutePath());
-			}
-			
-			isMaster = false;
-		}
-		
-		// attach uploaded files to records where possible
-		attachOpenUploads();
-		
 	}
 
 	private void attachOpenUploads() {
@@ -870,6 +945,10 @@ public class MamMigrationSelective implements CommandLineRunner {
 		adamUploadHeaders = createAdamHeaders();
 		adamUploadHeaders.set("Content-Type", "multipart/form-data");
 		adamUploadHeaders.set("Accept", "*/*");
+		adamSearchHeaders = createAdamHeaders();
+		adamSearchHeaders.set("sort", "createdon");
+		adamSearchHeaders.set("page", "1");
+		adamSearchHeaders.set("pageSize", "20");
 		mamHeaders = createMamHeaders();
 		
 		lastLogin = System.currentTimeMillis();
@@ -913,6 +992,13 @@ public class MamMigrationSelective implements CommandLineRunner {
 	private void invalidateAdamToken() {
 		ResponseEntity<String> response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
 		log.info("Logged out from adam API: " + response.toString());
+		
+		response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity<String>("", adamUploadHeaders), String.class);
+		log.info("Invalidated upload token: " + response.toString());
+		
+		response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, new HttpEntity<String>("", adamSearchHeaders), String.class);
+		log.info("Invalidated search token: " + response.toString());
+		
 	}
 	
 	/*