diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java b/Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java index 8e7fb35..dc4afb2 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java @@ -1,9 +1,11 @@ package com.bayer.edam.excelimport; import java.io.File; +import java.io.IOException; import java.util.HashSet; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Layout; @@ -24,10 +26,15 @@ import com.univocity.parsers.csv.CsvParserSettings; */ public class CropReleaseCopyTool { + private boolean dryRun = false; + private static Logger log = Logger.getLogger(CropReleaseCopyTool.class); private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.releasecopytool.log"; - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv"; + + private static String srcDir = "D:\\Toolbox\\ToolboxReleases\\"; + private static String targetDir = "D:\\ToolboxReleases\\"; private CsvParserSettings parserSettings = new CsvParserSettings(); private List inputRows; @@ -49,6 +56,20 @@ public class CropReleaseCopyTool { if (!copiedReleases.contains(releaseId)) { copiedReleases.add(releaseId); log.info(line[0] + " '" + line[1] + "' --> " + line[28] + " '" + line[29] + "'"); + + if (!dryRun) { + File srcFile = new File(srcDir + line[34]); + File targetFile = new File(targetDir + line[34]); + + try { + FileUtils.copyFile(srcFile, targetFile); + } + catch (IOException ioe) { + log.error("Line " + lineNo + ", AssetID " + line[0] + ", ReleaseID " + line[28] + " - IOException: '" + ioe.getMessage() + "'"); + ioe.printStackTrace(); + } + } + } } } diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java b/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java index 39284c8..b40a804 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java @@ -27,6 +27,7 @@ public class CropScienceAsset { HashSet brands = new HashSet(); HashSet indications = new HashSet(); + HashSet licenseRecords = new HashSet(); Long fileSize; diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/DirectoryListingCleaner.java b/Toolbox Migration/src/com/bayer/edam/excelimport/DirectoryListingCleaner.java new file mode 100644 index 0000000..d976c60 --- /dev/null +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/DirectoryListingCleaner.java @@ -0,0 +1,59 @@ +package com.bayer.edam.excelimport; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.RollingFileAppender; + +public class DirectoryListingCleaner { + + private static Logger log = Logger.getLogger(DirectoryListingCleaner.class); + private static String logfilePath = "D:\\ToolboxMigration-all.log"; + + private static String listingFilePath = "D:\\ToolboxMigration-all.txt"; + private List filenames; + HashSet uniqueFilenames = new HashSet(); + + public DirectoryListingCleaner() throws IOException { + filenames = FileUtils.readLines(new File(listingFilePath), "UTF-8"); + + int lineNo = 0; + for (String filename : filenames) { + lineNo++; + if (!uniqueFilenames.contains(filename)) { + uniqueFilenames.add(filename); + } + else { + log.warn("Filename '" + filename + "' sorted out (line " + lineNo + ")"); + } + } + + log.info("Unique filenames: " + uniqueFilenames.size()); + + } + + + public static void main(String[] args) { + try { + Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" + BasicConfigurator.configure(new ConsoleAppender(layout)); + BasicConfigurator.configure(new RollingFileAppender(layout, logfilePath, false)); + log.setLevel(Level.INFO); + + new DirectoryListingCleaner(); + + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java index b1be1e5..a67cdeb 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java @@ -29,13 +29,14 @@ import com.univocity.parsers.csv.CsvWriterSettings; public class ToolboxImportFileGenerator { private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class); - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv"; + private static String licenseGuidsFilePath = "C:\\Temp\\Migration CS - SQL Server\\Release_GUIDs.csv"; // XXX Spreadsheet needs be be named "Records" in Excel for the importer - private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files-IMPORT.csv"; + private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases-IMPORT.csv"; - private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.log"; - private static String uploadPath = "/Resources/Upload/ToolboxMigration2"; + private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.log"; + private static String uploadPath = "/Resources/Upload/ToolboxMigration"; private static final Map brandMap; private static final Map unitMap; @@ -974,6 +975,7 @@ public class ToolboxImportFileGenerator { } private List inputRows; + private HashMap licenses = new HashMap(); private HashMap assets = new HashMap(); private CsvParserSettings parserSettings = new CsvParserSettings(); @@ -981,7 +983,26 @@ public class ToolboxImportFileGenerator { private SimpleDateFormat outputDateFormat = new SimpleDateFormat("MM/dd/yyyy"); public ToolboxImportFileGenerator() throws Exception { - readToolboxExport(); + parserSettings.getFormat().setLineSeparator("\r\n"); + parserSettings.getFormat().setDelimiter(';'); + parserSettings.getFormat().setCharToEscapeQuoteEscaping('"'); + parserSettings.setHeaderExtractionEnabled(false); + + CsvParser parser = new CsvParser(parserSettings); + + inputRows = parser.parseAll(new File(toolboxExportFilePath)); + + List licenseGuidRows = parser.parseAll(new File(licenseGuidsFilePath)); + for (String[] line : licenseGuidRows) { + if (!licenses.containsKey(line[1])) { + licenses.put(line[1], line[0]); + } + else { + log.error("License " + line[1] + " has multiple keys"); + } + } + + generateAssetDecriptions(); writeImportFile(); } @@ -998,6 +1019,7 @@ public class ToolboxImportFileGenerator { long internalAssets = 0; long statusSkipped = 0; long assetTypeNull = 0; + long releaseMissing = 0; for (String[] line : inputRows) { Integer assetId = Integer.parseInt(line[0]); @@ -1152,9 +1174,18 @@ public class ToolboxImportFileGenerator { // + " of already assigned brand(s) " + assignedBrands); // } - // Indications stored in POJO for reference only. Indications are added to classifications in readToolboxExport() if (indicationMap.get(line[14]) != null) asset.indications.add(indicationMap.get(line[14])); + + if (!line[34].equals("NULL")) { + if (licenses.containsKey(line[34])) { + asset.licenseRecords.add(licenses.get(line[34])); + } + else { + log.error(asset.toolboxAssetId + ", line " + lineNo + ": No GUID for release file " + line[34]); + releaseMissing++; + } + } } @@ -1166,21 +1197,11 @@ public class ToolboxImportFileGenerator { log.info("Lines Skipped: No target business unit: " + noTargetBusinessUnit); log.info("Lines Skipped: Status is not 'Publish' or 'Archive': " + statusSkipped); log.info("Lines Skipped: Asset type NULL: " + assetTypeNull); + log.info("Release missing: " + releaseMissing); log.info("No asset type: " + noAssetType); log.info("Internal assets: " + internalAssets); } - private void readToolboxExport() { - parserSettings.getFormat().setLineSeparator("\r\n"); - parserSettings.getFormat().setDelimiter(';'); - parserSettings.getFormat().setCharToEscapeQuoteEscaping('"'); - parserSettings.setHeaderExtractionEnabled(false); - - CsvParser parser = new CsvParser(parserSettings); - - inputRows = parser.parseAll(new File(toolboxExportFilePath)); - } - private void writeImportFile() { List csvOutputList = new ArrayList(); for (CropScienceAsset asset : assets.values()) { @@ -1191,14 +1212,22 @@ public class ToolboxImportFileGenerator { brands += "\n"; brands += brand; } + for (String indication : asset.indications) asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication; - + + String licenseRecords = ""; + for (String licenseRecord : asset.licenseRecords) { + if (licenseRecords.length() > 0) + licenseRecords += "\n"; + licenseRecords += licenseRecord; + } + String[] csvLine = new String[] { asset.assetPath, asset.filename, asset.appendClassification, asset.byr_Title, asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_businessUnit, brands, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, asset.byr_cs_keywords_tl, asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished, - asset.byr_cs_warehouseAvailable, asset.byr_cs_country }; + asset.byr_cs_warehouseAvailable, asset.byr_cs_country, licenseRecords }; csvOutputList.add(csvLine); } @@ -1210,7 +1239,7 @@ public class ToolboxImportFileGenerator { "fields|byr_cs_assetApproval", "fields|byr_cs_licenseType", "fields|byr_cs_businessUnit", "fields|byr_cs_brand", "fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl", "fields|byr_campaignYearLabel", "fields|byr_campaignTitleLabel", "fields|byr_cs_IsShouldBePublished", - "fields|byr_cs_warehouseAvailable", "fields|byr_cs_country"); + "fields|byr_cs_warehouseAvailable", "fields|byr_cs_country", "fields|byr_cs_licenseRecord"); writer.writeRowsAndClose(csvOutputList); }