tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed duplicate keywords (e.g. "Myiases", "myiases ")

Browse Source
This commit is contained in:
GEXCE
2017-11-08 22:47:54 +01:00
parent c0ffd9fc20
commit a129c124b7
1 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+7 -6
View File
@@ -15,7 +15,6 @@ import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@@ -179,7 +178,7 @@ public class MamMigrationSelective implements CommandLineRunner {
void postToAdam(Entry entry) throws Exception { void postToAdam(Entry entry) throws Exception {
UploadRecord record = new UploadRecord(); UploadRecord record = new UploadRecord();
String entryJson = jsonMapper.writeValueAsString(entry); String entryJson = jsonMapper.writeValueAsString(entry);
Set<String> keywords = new HashSet<String>(); List<String> keywords = new ArrayList<String>();
// globally applicable classifications // globally applicable classifications
record.addClassification("Migration"); record.addClassification("Migration");
@@ -423,9 +422,12 @@ public class MamMigrationSelective implements CommandLineRunner {
// Keywords // Keywords
if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) { if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*")); List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
HashSet<String> keywordSet = new HashSet<String>();
for (String s : mamKeywords) { for (String s : mamKeywords) {
keywords.add(s); if (keywordSet.add(s.trim().toLowerCase())) {
keywords.add(s.trim());
}
// if the keyword is four digits - dash - four digits, it's a GPC number // if the keyword is four digits - dash - four digits, it's a GPC number
if (s.matches("\\b\\d{4}-\\d{4}\\b")) if (s.matches("\\b\\d{4}-\\d{4}\\b"))
@@ -434,7 +436,7 @@ public class MamMigrationSelective implements CommandLineRunner {
} }
if (keywords.size() > 0) { if (keywords.size() > 0) {
record.addField("byr_ah_keywords_tl", new ArrayList<String>(keywords)); record.addField("byr_ah_keywords_tl", keywords);
} }
// License Information // License Information
@@ -570,8 +572,7 @@ public class MamMigrationSelective implements CommandLineRunner {
} }
else { else {
// TODO // TODO
log.debug("File too large, could not upload " + file.getName() + " for " + recordLocation);
log.debug("filesize " + file.length() + " > 4 GB!!!1!");
} }
} }
else { else {