From c881e7490b3988b30a78d36b424fb446d53e021a Mon Sep 17 00:00:00 2001 From: gexce Date: Tue, 22 Aug 2017 11:59:20 +0200 Subject: [PATCH] final fixes --- .../src/com/bayer/edam/ToolboxFileExporter.java | 8 ++++---- .../edam/excelimport/ImportFileCleaner.java | 16 ++++++++++------ .../excelimport/ToolboxImportFileGenerator.java | 13 +++++++------ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java index 82149fd..745fb3d 100644 --- a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java +++ b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java @@ -22,7 +22,7 @@ public class ToolboxFileExporter { private List rows; - private String baseDir = "D:\\Toolbox\\Assets"; + private String baseDir = "D:\\ToolboxMigration_missing"; // Formatdatei generiert mit // bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T @@ -54,7 +54,7 @@ public class ToolboxFileExporter { } else { log.info("Download for asset " + assetId + ", OriginalFilename: '" + originalFilenname + "'"); - Process process = Runtime.getRuntime().exec(command1 + assetId + command2 + assetId + command3, null, workingDir); + Process process = Runtime.getRuntime().exec(command1 + assetId + command2 + originalFilenname + command3, null, workingDir); process.waitFor(); if (0 < process.exitValue()) { BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); @@ -75,7 +75,7 @@ public class ToolboxFileExporter { CsvParser parser = new CsvParser(settings); - rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv")); + rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\missing_after_clean.csv")); } @@ -83,7 +83,7 @@ public class ToolboxFileExporter { try { Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" BasicConfigurator.configure(new ConsoleAppender(layout)); - BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\Toolbox\\download.log", true)); + BasicConfigurator.configure(new RollingFileAppender(layout, "C:\\Temp\\Migration CS - SQL Server\\missing_after_clean.log", false)); log.setLevel(Level.DEBUG); long startTime = System.currentTimeMillis(); diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ImportFileCleaner.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ImportFileCleaner.java index 1a3c4ca..8ad3442 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/ImportFileCleaner.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ImportFileCleaner.java @@ -20,8 +20,7 @@ import com.univocity.parsers.csv.CsvWriter; import com.univocity.parsers.csv.CsvWriterSettings; /** - * Quick and dirty: Remove all files from the import Excel spreadsheet that are - * not shown in a directory listing. + * Removes all files from the import Excel spreadsheet that are not shown in a directory listing. * * @author gezvf * @@ -32,13 +31,14 @@ public class ImportFileCleaner { private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv"; private static String listingFilePath = "D:\\largeListing.txt"; - private static String outputFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet.csv"; - private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_clean.log"; + private static String outputFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_2.csv"; + private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_clean_2.log"; private CsvParserSettings parserSettings = new CsvParserSettings(); private List inputRows; private List filenames; private List linesToKeep = new ArrayList(); + private List linesToRemove = new ArrayList(); public ImportFileCleaner() throws IOException { parserSettings.getFormat().setLineSeparator("\r\n"); @@ -53,6 +53,10 @@ public class ImportFileCleaner { for (String[] line : inputRows) { if (filenames.contains(line[1])) linesToKeep.add(line); + else { + linesToRemove.add(line); + log.info("Asset ID " + line[0] + " - File '" + line[1] + "' not found, removing from import"); + } } List csvOutputList = new ArrayList(); @@ -66,8 +70,8 @@ public class ImportFileCleaner { CsvWriter writer = new CsvWriter(new File(outputFilePath), writerSettings); writer.writeRowsAndClose(csvOutputList); - log.info("No input lines: " + inputRows.size()); - log.info("No output lines: " + linesToKeep.size()); + log.info("Number of input lines: " + inputRows.size()); + log.info("Number of output lines: " + linesToKeep.size()); } diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java index 15ad7ee..3da18f4 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java @@ -29,12 +29,13 @@ import com.univocity.parsers.csv.CsvWriterSettings; public class ToolboxImportFileGenerator { private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class); - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-leftover.csv"; // XXX Spreadsheet needs be be named "Records" in Excel for the importer - private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod.csv"; + private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod_LEFTOVER.csv"; - private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod.log"; + private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod_LEFTOVER.log"; + private static String uploadPath = "/Resources/Upload/ToolboxMigration"; private static final Map brandMap; private static final Map unitMap; @@ -213,8 +214,8 @@ public class ToolboxImportFileGenerator { brandMap.put("Prosper FX", "Prosper"); brandMap.put("Protech Plus", "TEMPORARY"); brandMap.put("ProTechPlus", "TEMPORARY"); - brandMap.put("Prot�g�", "TEMPORARY"); - brandMap.put("Prot�g� FL", "TEMPORARY"); + brandMap.put("Prot�g�", "TEMPORARY"); // XXX + brandMap.put("Prot�g� FL", "TEMPORARY"); // XXX brandMap.put("Provado", "TEMPORARY"); brandMap.put("Provost", "Provost"); brandMap.put("Provost Opti", "Provost Opti"); @@ -1042,7 +1043,7 @@ public class ToolboxImportFileGenerator { assetsIncluded++; asset = new CropScienceAsset(assetId); - asset.assetPath = "/Resources/Upload/ToolboxMigration"; + asset.assetPath = uploadPath; asset.filename = line[1]; asset.byr_Title = line[2];