diff --git a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java index 2b0ae40..e1a45e4 100644 --- a/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java +++ b/AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java @@ -15,7 +15,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; @@ -179,7 +178,7 @@ public class MamMigrationSelective implements CommandLineRunner { void postToAdam(Entry entry) throws Exception { UploadRecord record = new UploadRecord(); String entryJson = jsonMapper.writeValueAsString(entry); - Set keywords = new HashSet(); + List keywords = new ArrayList(); // globally applicable classifications record.addClassification("Migration"); @@ -423,10 +422,13 @@ public class MamMigrationSelective implements CommandLineRunner { // Keywords if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) { List mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")); + HashSet keywordSet = new HashSet(); for (String s : mamKeywords) { - keywords.add(s); - + if (keywordSet.add(s.trim().toLowerCase())) { + keywords.add(s.trim()); + } + // if the keyword is four digits - dash - four digits, it's a GPC number if (s.matches("\\b\\d{4}-\\d{4}\\b")) record.addField("byr_ah_approvalNumber", s); @@ -434,7 +436,7 @@ public class MamMigrationSelective implements CommandLineRunner { } if (keywords.size() > 0) { - record.addField("byr_ah_keywords_tl", new ArrayList(keywords)); + record.addField("byr_ah_keywords_tl", keywords); } // License Information @@ -570,8 +572,7 @@ public class MamMigrationSelective implements CommandLineRunner { } else { // TODO - - log.debug("filesize " + file.length() + " > 4 GB!!!1!"); + log.debug("File too large, could not upload " + file.getName() + " for " + recordLocation); } } else {