tilman/mam-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Campaigns, first version: storing in CSV file

Browse Source
This commit is contained in:
gexce
2017-10-20 17:10:40 +02:00
parent 885b38bb6c
commit b767f1b8af
8 changed files with 11474 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH MAM Migration/src/com/bayer/edam/migration/MamMigrationSelective.java
+116 -20
View File
@@ -30,6 +30,8 @@ import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestTemplate;
import com.bayer.edam.migration.adam.Campaign;
import com.bayer.edam.migration.adam.EntityId;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.WriteRecord;
import com.bayer.edam.migration.adam.WriteRecord.Classification;
@@ -37,6 +39,10 @@ import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.categories.CategoryTree;
import com.bayer.edam.migration.mam.categories.CategoryTree.Node;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
import com.univocity.parsers.csv.CsvWriter;
import com.univocity.parsers.csv.CsvWriterSettings;
@SpringBootApplication
@PropertySource("file:secret.properties")
@@ -48,9 +54,14 @@ public class MamMigrationSelective implements CommandLineRunner {
static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
static final int[] selection = { 71948, /*71048, 70778, 70746, 70142,*/ 70361, /*63924, 71263, 63201, 56942, 57641, 70745, 70440, 70340,
70288, 71725*/ };
static final File entryLog = new File("C:\\Temp\\Migration AH\\entry.log");
static final File problemLog = new File("C:\\Temp\\Migration AH\\problem.log");
static final String categoryJson = "C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\2017-10-19_categories.json";
static final String categoryMapping = "C:\\Users\\gexce\\git\\mam-migration\\AH MAM Migration\\resources\\Media_Assistant_Categories_131017_EDITED.csv";
static final String campaignList = "C:\\Temp\\Migration AH\\files\\campaigns.csv";
static final File entryLog = new File("C:\\Temp\\Migration AH\\files\\entry.log");
static final File problemLog = new File("C:\\Temp\\Migration AH\\files\\problem.log");
static final String fieldSeparator = "|~|";
static final String lineSeparator = "\r\n";
@@ -68,18 +79,29 @@ public class MamMigrationSelective implements CommandLineRunner {
private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
private ObjectMapper jsonMapper = new ObjectMapper();
private List<String[]> campaigns;
private List<String> locations = new ArrayList<String>();
private CategoryTree mamCategoryTree;
private CsvParserSettings parserSettings = new CsvParserSettings();
@Override
public void run(String... strings) throws Exception {
parserSettings.getFormat().setLineSeparator("\r\n");
parserSettings.getFormat().setDelimiter(';');
parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
parserSettings.setHeaderExtractionEnabled(false);
log.info("Write log headers");
FileUtils.writeStringToFile(entryLog, "Location" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
FileUtils.writeStringToFile(problemLog, "entryId" + fieldSeparator + "Entry" + lineSeparator, Charset.forName("utf-8"));
log.info("Read categories");
mamCategoryTree = new CategoryTree("C:\\Temp\\Migration AH\\2017-10-19_categories.json", "C:\\Temp\\Migration AH\\Media_Assistant_Categories_131017_EDITED.csv");
mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
log.info("Read campaigns");
campaigns = readCsv(campaignList);
restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
@@ -119,6 +141,9 @@ public class MamMigrationSelective implements CommandLineRunner {
// String.class);
// log.info("Deleted " + location + ": " + response.toString());
// }
log.info("Write campaign list to " + campaignList);
writeCsv(campaigns, campaignList);
invalidateAdamToken();
}
@@ -168,7 +193,52 @@ public class MamMigrationSelective implements CommandLineRunner {
record.addClassification(brandClassificationId);
}
else if (n.action.equals("campaign")) {
String campaignName = n.name;
String campaignLabel = n.name;
String campaignYear;
if (n.parameter != null && !n.parameter.trim().equals("")) {
campaignLabel = campaignLabel + " " + n.parameter;
campaignYear = n.parameter.substring(1, 5);
} else {
campaignYear = n.name.substring(0, 4);
}
try {
Integer.parseInt(campaignYear);
// check whether campain already exists
String campaignGuid = null;
for (String[] line : campaigns) {
if (line[0].equals(campaignName)) {
campaignGuid = line[2];
log.debug("Found already existing campaign '" + campaignName + "' with guid " + campaignGuid);
}
}
if (campaignGuid == null) {
log.info("Create campaign " + campaignName);
Campaign campaign = new Campaign(campaignName, campaignLabel, campaignYear);
HttpEntity<Campaign> httpEntity = new HttpEntity<Campaign>(campaign, adamHeaders);
try {
EntityId campaignId = restTemplate.exchange(adamHost + "/classifications", HttpMethod.POST, httpEntity, EntityId.class).getBody();
campaignGuid = campaignId.id;
log.info("Created campaign: " + campaignName + ";" + campaignLabel + ";" + campaignGuid);
record.addClassification(new Classification(campaignGuid, 1));
// TODO better write campaign to file right away?
campaigns.add(new String[] { campaignName, campaignLabel, campaignGuid });
} catch (HttpStatusCodeException hsce) {
log.error("Could not create campaign '" + campaignName + "' for entry " + entry.id + ": " + hsce.getStatusCode().toString());
log.error(hsce.getResponseBodyAsString());
}
}
} catch (NumberFormatException nfe) {
log.warn("Could not parse year from string " + campaignYear);
}
}
else if (n.action.equals("keyword")) {
log.debug("Adding keyword " + n.name.trim());
@@ -229,21 +299,30 @@ public class MamMigrationSelective implements CommandLineRunner {
// Create record
// HttpEntity<WriteRecord> httpEntity = new HttpEntity<WriteRecord>(record, adamHeaders);
// try {
// URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class);
// log.info("Created " + location);
//
// FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
// Charset.forName("utf-8"), true);
//
// locations.add(location.toString());
//
// } catch (HttpStatusCodeException hsce) {
// log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
// log.error(hsce.getResponseBodyAsString());
// FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
// }
HttpEntity<WriteRecord> httpEntity = new HttpEntity<WriteRecord>(record, adamHeaders);
try {
URI location = restTemplate.postForLocation(adamHost + "/records", httpEntity, WriteRecord.class);
log.info("Created " + location);
FileUtils.writeStringToFile(entryLog, location.toString() + fieldSeparator + entryJson + lineSeparator,
Charset.forName("utf-8"), true);
locations.add(location.toString());
} catch (HttpStatusCodeException hsce) {
log.error("Skipping entry " + entry.id + ": Error while creating record " + hsce.getStatusCode().toString());
log.error(hsce.getResponseBodyAsString());
FileUtils.writeStringToFile(problemLog, entry.id + fieldSeparator + entryJson + lineSeparator, Charset.forName("utf-8"), true);
}
}
/**
* Checks whether the given campaign already exists and creates it, if not.
*
* @return the guid of the campaign
*/
private String getCampaign(int year, String label, String name) {
return null;
}
private HttpHeaders createMamHeaders() {
@@ -281,6 +360,23 @@ public class MamMigrationSelective implements CommandLineRunner {
ResponseEntity<String> response = restTemplate.exchange(adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
log.info("Logged out from adam API: " + response.toString());
}
private List<String[]> readCsv(String csvFile) {
CsvParser parser = new CsvParser(parserSettings);
return parser.parseAll(new File(csvFile));
}
private void writeCsv(List<String[]> contents, String csvFile) {
// FIXME Why oh why can't I just pass the contents list to writeRowsAndClose like here:
// https://github.com/uniVocity/univocity-parsers/blob/master/src/test/java/com/univocity/parsers/examples/WriterExamples.java
List<Object[]> csvOutputList;
csvOutputList = (List) contents;
CsvWriterSettings writerSettings = new CsvWriterSettings();
writerSettings.setFormat(parserSettings.getFormat());
CsvWriter writer = new CsvWriter(new File(csvFile), writerSettings);
writer.writeRowsAndClose(csvOutputList);
}
/*
* // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new