tilman/toolbox-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding release files

Browse Source
This commit is contained in:
gexce
2017-08-24 11:35:37 +02:00
parent be332f977c
commit 26f3598bf1
4 changed files with 131 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java
+22 -1
View File
@@ -1,9 +1,11 @@
package com.bayer.edam.excelimport;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
@@ -24,10 +26,15 @@ import com.univocity.parsers.csv.CsvParserSettings;
*/
public class CropReleaseCopyTool {
private boolean dryRun = false;
private static Logger log = Logger.getLogger(CropReleaseCopyTool.class);
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.releasecopytool.log";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.csv";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv";
private static String srcDir = "D:\\Toolbox\\ToolboxReleases\\";
private static String targetDir = "D:\\ToolboxReleases\\";
private CsvParserSettings parserSettings = new CsvParserSettings();
private List<String[]> inputRows;
@@ -49,6 +56,20 @@ public class CropReleaseCopyTool {
if (!copiedReleases.contains(releaseId)) {
copiedReleases.add(releaseId);
log.info(line[0] + " '" + line[1] + "' --> " + line[28] + " '" + line[29] + "'");
if (!dryRun) {
File srcFile = new File(srcDir + line[34]);
File targetFile = new File(targetDir + line[34]);
try {
FileUtils.copyFile(srcFile, targetFile);
}
catch (IOException ioe) {
log.error("Line " + lineNo + ", AssetID " + line[0] + ", ReleaseID " + line[28] + " - IOException: '" + ioe.getMessage() + "'");
ioe.printStackTrace();
}
}
}
}
}
Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java
+1
View File
@@ -27,6 +27,7 @@ public class CropScienceAsset {
HashSet<String> brands = new HashSet<String>();
HashSet<String> indications = new HashSet<String>();
HashSet<String> licenseRecords = new HashSet<String>();
Long fileSize;
Toolbox Migration/src/com/bayer/edam/excelimport/DirectoryListingCleaner.java
+59
View File
@@ -0,0 +1,59 @@
package com.bayer.edam.excelimport;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
public class DirectoryListingCleaner {
private static Logger log = Logger.getLogger(DirectoryListingCleaner.class);
private static String logfilePath = "D:\\ToolboxMigration-all.log";
private static String listingFilePath = "D:\\ToolboxMigration-all.txt";
private List<String> filenames;
HashSet<String> uniqueFilenames = new HashSet<String>();
public DirectoryListingCleaner() throws IOException {
filenames = FileUtils.readLines(new File(listingFilePath), "UTF-8");
int lineNo = 0;
for (String filename : filenames) {
lineNo++;
if (!uniqueFilenames.contains(filename)) {
uniqueFilenames.add(filename);
}
else {
log.warn("Filename '" + filename + "' sorted out (line " + lineNo + ")");
}
}
log.info("Unique filenames: " + uniqueFilenames.size());
}
public static void main(String[] args) {
try {
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, logfilePath, false));
log.setLevel(Level.INFO);
new DirectoryListingCleaner();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java
+48 -19
View File
@@ -29,13 +29,14 @@ import com.univocity.parsers.csv.CsvWriterSettings;
public class ToolboxImportFileGenerator {
private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class);
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.csv";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv";
private static String licenseGuidsFilePath = "C:\\Temp\\Migration CS - SQL Server\\Release_GUIDs.csv";
// XXX Spreadsheet needs be be named "Records" in Excel for the importer
private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files-IMPORT.csv";
private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases-IMPORT.csv";
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.log";
private static String uploadPath = "/Resources/Upload/ToolboxMigration2";
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.log";
private static String uploadPath = "/Resources/Upload/ToolboxMigration";
private static final Map<String, String> brandMap;
private static final Map<String, String> unitMap;
@@ -974,6 +975,7 @@ public class ToolboxImportFileGenerator {
}
private List<String[]> inputRows;
private HashMap<String, String> licenses = new HashMap<String, String>();
private HashMap<Integer, CropScienceAsset> assets = new HashMap<Integer, CropScienceAsset>();
private CsvParserSettings parserSettings = new CsvParserSettings();
@@ -981,7 +983,26 @@ public class ToolboxImportFileGenerator {
private SimpleDateFormat outputDateFormat = new SimpleDateFormat("MM/dd/yyyy");
public ToolboxImportFileGenerator() throws Exception {
readToolboxExport();
parserSettings.getFormat().setLineSeparator("\r\n");
parserSettings.getFormat().setDelimiter(';');
parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
parserSettings.setHeaderExtractionEnabled(false);
CsvParser parser = new CsvParser(parserSettings);
inputRows = parser.parseAll(new File(toolboxExportFilePath));
List<String[]> licenseGuidRows = parser.parseAll(new File(licenseGuidsFilePath));
for (String[] line : licenseGuidRows) {
if (!licenses.containsKey(line[1])) {
licenses.put(line[1], line[0]);
}
else {
log.error("License " + line[1] + " has multiple keys");
}
}
generateAssetDecriptions();
writeImportFile();
}
@@ -998,6 +1019,7 @@ public class ToolboxImportFileGenerator {
long internalAssets = 0;
long statusSkipped = 0;
long assetTypeNull = 0;
long releaseMissing = 0;
for (String[] line : inputRows) {
Integer assetId = Integer.parseInt(line[0]);
@@ -1152,10 +1174,19 @@ public class ToolboxImportFileGenerator {
// + " of already assigned brand(s) " + assignedBrands);
// }
// Indications stored in POJO for reference only. Indications are added to classifications in readToolboxExport()
if (indicationMap.get(line[14]) != null)
asset.indications.add(indicationMap.get(line[14]));
if (!line[34].equals("NULL")) {
if (licenses.containsKey(line[34])) {
asset.licenseRecords.add(licenses.get(line[34]));
}
else {
log.error(asset.toolboxAssetId + ", line " + lineNo + ": No GUID for release file " + line[34]);
releaseMissing++;
}
}
}
log.info("Processed lines : " + lineNo);
@@ -1166,21 +1197,11 @@ public class ToolboxImportFileGenerator {
log.info("Lines Skipped: No target business unit: " + noTargetBusinessUnit);
log.info("Lines Skipped: Status is not 'Publish' or 'Archive': " + statusSkipped);
log.info("Lines Skipped: Asset type NULL: " + assetTypeNull);
log.info("Release missing: " + releaseMissing);
log.info("No asset type: " + noAssetType);
log.info("Internal assets: " + internalAssets);
}
private void readToolboxExport() {
parserSettings.getFormat().setLineSeparator("\r\n");
parserSettings.getFormat().setDelimiter(';');
parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
parserSettings.setHeaderExtractionEnabled(false);
CsvParser parser = new CsvParser(parserSettings);
inputRows = parser.parseAll(new File(toolboxExportFilePath));
}
private void writeImportFile() {
List<Object[]> csvOutputList = new ArrayList<Object[]>();
for (CropScienceAsset asset : assets.values()) {
@@ -1191,14 +1212,22 @@ public class ToolboxImportFileGenerator {
brands += "\n";
brands += brand;
}
for (String indication : asset.indications)
asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication;
String licenseRecords = "";
for (String licenseRecord : asset.licenseRecords) {
if (licenseRecords.length() > 0)
licenseRecords += "\n";
licenseRecords += licenseRecord;
}
String[] csvLine = new String[] { asset.assetPath, asset.filename, asset.appendClassification, asset.byr_Title,
asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_businessUnit,
brands, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, asset.byr_cs_keywords_tl,
asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished,
asset.byr_cs_warehouseAvailable, asset.byr_cs_country };
asset.byr_cs_warehouseAvailable, asset.byr_cs_country, licenseRecords };
csvOutputList.add(csvLine);
}
@@ -1210,7 +1239,7 @@ public class ToolboxImportFileGenerator {
"fields|byr_cs_assetApproval", "fields|byr_cs_licenseType", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
"fields|byr_campaignYearLabel", "fields|byr_campaignTitleLabel", "fields|byr_cs_IsShouldBePublished",
"fields|byr_cs_warehouseAvailable", "fields|byr_cs_country");
"fields|byr_cs_warehouseAvailable", "fields|byr_cs_country", "fields|byr_cs_licenseRecord");
writer.writeRowsAndClose(csvOutputList);
}