tilman/toolbox-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

final fixes

Browse Source
This commit is contained in:
gexce
2017-08-22 11:59:20 +02:00
parent aacaad6ced
commit c881e7490b
3 changed files with 21 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java
+4 -4
View File
@@ -22,7 +22,7 @@ public class ToolboxFileExporter {
private List<String[]> rows; private List<String[]> rows;
private String baseDir = "D:\\Toolbox\\Assets"; private String baseDir = "D:\\ToolboxMigration_missing";
// Formatdatei generiert mit // Formatdatei generiert mit
// bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T // bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T
@@ -54,7 +54,7 @@ public class ToolboxFileExporter {
} }
else { else {
log.info("Download for asset " + assetId + ", OriginalFilename: '" + originalFilenname + "'"); log.info("Download for asset " + assetId + ", OriginalFilename: '" + originalFilenname + "'");
Process process = Runtime.getRuntime().exec(command1 + assetId + command2 + assetId + command3, null, workingDir); Process process = Runtime.getRuntime().exec(command1 + assetId + command2 + originalFilenname + command3, null, workingDir);
process.waitFor(); process.waitFor();
if (0 < process.exitValue()) { if (0 < process.exitValue()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
@@ -75,7 +75,7 @@ public class ToolboxFileExporter {
CsvParser parser = new CsvParser(settings); CsvParser parser = new CsvParser(settings);
rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv")); rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\missing_after_clean.csv"));
} }
@@ -83,7 +83,7 @@ public class ToolboxFileExporter {
try { try {
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout)); BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\Toolbox\\download.log", true)); BasicConfigurator.configure(new RollingFileAppender(layout, "C:\\Temp\\Migration CS - SQL Server\\missing_after_clean.log", false));
log.setLevel(Level.DEBUG); log.setLevel(Level.DEBUG);
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
Toolbox Migration/src/com/bayer/edam/excelimport/ImportFileCleaner.java
+10 -6
View File
@@ -20,8 +20,7 @@ import com.univocity.parsers.csv.CsvWriter;
import com.univocity.parsers.csv.CsvWriterSettings; import com.univocity.parsers.csv.CsvWriterSettings;
/** /**
* Quick and dirty: Remove all files from the import Excel spreadsheet that are * Removes all files from the import Excel spreadsheet that are not shown in a directory listing.
* not shown in a directory listing.
* *
* @author gezvf * @author gezvf
* *
@@ -32,13 +31,14 @@ public class ImportFileCleaner {
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv"; private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv";
private static String listingFilePath = "D:\\largeListing.txt"; private static String listingFilePath = "D:\\largeListing.txt";
private static String outputFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet.csv"; private static String outputFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_2.csv";
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_clean.log"; private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet_clean_2.log";
private CsvParserSettings parserSettings = new CsvParserSettings(); private CsvParserSettings parserSettings = new CsvParserSettings();
private List<String[]> inputRows; private List<String[]> inputRows;
private List<String> filenames; private List<String> filenames;
private List<String[]> linesToKeep = new ArrayList<String[]>(); private List<String[]> linesToKeep = new ArrayList<String[]>();
private List<String[]> linesToRemove = new ArrayList<String[]>();
public ImportFileCleaner() throws IOException { public ImportFileCleaner() throws IOException {
parserSettings.getFormat().setLineSeparator("\r\n"); parserSettings.getFormat().setLineSeparator("\r\n");
@@ -53,6 +53,10 @@ public class ImportFileCleaner {
for (String[] line : inputRows) { for (String[] line : inputRows) {
if (filenames.contains(line[1])) if (filenames.contains(line[1]))
linesToKeep.add(line); linesToKeep.add(line);
else {
linesToRemove.add(line);
log.info("Asset ID " + line[0] + " - File '" + line[1] + "' not found, removing from import");
}
} }
List<Object[]> csvOutputList = new ArrayList<Object[]>(); List<Object[]> csvOutputList = new ArrayList<Object[]>();
@@ -66,8 +70,8 @@ public class ImportFileCleaner {
CsvWriter writer = new CsvWriter(new File(outputFilePath), writerSettings); CsvWriter writer = new CsvWriter(new File(outputFilePath), writerSettings);
writer.writeRowsAndClose(csvOutputList); writer.writeRowsAndClose(csvOutputList);
log.info("No input lines: " + inputRows.size()); log.info("Number of input lines: " + inputRows.size());
log.info("No output lines: " + linesToKeep.size()); log.info("Number of output lines: " + linesToKeep.size());
} }
Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java
+7 -6
View File
@@ -29,12 +29,13 @@ import com.univocity.parsers.csv.CsvWriterSettings;
public class ToolboxImportFileGenerator { public class ToolboxImportFileGenerator {
private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class); private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class);
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-largeSet.csv"; private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update-leftover.csv";
// XXX Spreadsheet needs be be named "Records" in Excel for the importer // XXX Spreadsheet needs be be named "Records" in Excel for the importer
private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod.csv"; private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod_LEFTOVER.csv";
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod.log"; private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile-prod_LEFTOVER.log";
private static String uploadPath = "/Resources/Upload/ToolboxMigration";
private static final Map<String, String> brandMap; private static final Map<String, String> brandMap;
private static final Map<String, String> unitMap; private static final Map<String, String> unitMap;
@@ -213,8 +214,8 @@ public class ToolboxImportFileGenerator {
brandMap.put("Prosper FX", "Prosper"); brandMap.put("Prosper FX", "Prosper");
brandMap.put("Protech Plus", "TEMPORARY"); brandMap.put("Protech Plus", "TEMPORARY");
brandMap.put("ProTechPlus", "TEMPORARY"); brandMap.put("ProTechPlus", "TEMPORARY");
brandMap.put("Prot�g�", "TEMPORARY"); brandMap.put("Prot�g�", "TEMPORARY"); // XXX
brandMap.put("Prot�g� FL", "TEMPORARY"); brandMap.put("Prot�g� FL", "TEMPORARY"); // XXX
brandMap.put("Provado", "TEMPORARY"); brandMap.put("Provado", "TEMPORARY");
brandMap.put("Provost", "Provost"); brandMap.put("Provost", "Provost");
brandMap.put("Provost Opti", "Provost Opti"); brandMap.put("Provost Opti", "Provost Opti");
@@ -1042,7 +1043,7 @@ public class ToolboxImportFileGenerator {
assetsIncluded++; assetsIncluded++;
asset = new CropScienceAsset(assetId); asset = new CropScienceAsset(assetId);
asset.assetPath = "/Resources/Upload/ToolboxMigration"; asset.assetPath = uploadPath;
asset.filename = line[1]; asset.filename = line[1];
asset.byr_Title = line[2]; asset.byr_Title = line[2];