Removed duplicate keywords (e.g. "Myiases", "myiases ")

AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java

@@ -15,7 +15,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.FileUtils;
@@ -179,7 +178,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 	void postToAdam(Entry entry) throws Exception {
 		UploadRecord record = new UploadRecord();
 		String entryJson = jsonMapper.writeValueAsString(entry);
-		Set<String> keywords = new HashSet<String>();
+		List<String> keywords = new ArrayList<String>();
 
 		// globally applicable classifications
 		record.addClassification("Migration");
@@ -423,10 +422,13 @@ public class MamMigrationSelective implements CommandLineRunner {
 		// Keywords
 		if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
 			List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
+			HashSet<String> keywordSet = new HashSet<String>();
 			
 			for (String s : mamKeywords) {
-				keywords.add(s);
-				
+				if (keywordSet.add(s.trim().toLowerCase())) {
+					keywords.add(s.trim());
+				}
+					
 				// if the keyword is four digits - dash - four digits, it's a GPC number
 				if (s.matches("\\b\\d{4}-\\d{4}\\b"))
 					record.addField("byr_ah_approvalNumber", s);
@@ -434,7 +436,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 		}
 		
 		if (keywords.size() > 0) {
-			record.addField("byr_ah_keywords_tl", new ArrayList<String>(keywords));
+			record.addField("byr_ah_keywords_tl", keywords);
 		}
 		
 		// License Information
@@ -570,8 +572,7 @@ public class MamMigrationSelective implements CommandLineRunner {
 				}
 				else {
 					// TODO
-					
-					log.debug("filesize " + file.length() + " > 4 GB!!!1!");
+					log.debug("File too large, could not upload " + file.getName() + " for " + recordLocation);
 				}
 			}
 			else {