From df8727f9b8ec7886126f8c36b8de1f89acdbbce1 Mon Sep 17 00:00:00 2001 From: gexce Date: Mon, 28 Aug 2017 14:23:05 +0200 Subject: [PATCH] ImportVerifier --- .../com/bayer/edam/ToolboxFileExporter.java | 6 +- .../edam/excelimport/ImportVerifier.java | 95 +++++++++++++++++++ .../ToolboxImportFileGenerator.java | 23 +---- .../com/bayer/edam/excelimport/sql-code.sql | 3 + 4 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 Toolbox Migration/src/com/bayer/edam/excelimport/ImportVerifier.java diff --git a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java index 80cf140..ad5c4dc 100644 --- a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java +++ b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java @@ -19,12 +19,12 @@ import com.univocity.parsers.csv.CsvParserSettings; public class ToolboxFileExporter { private static Logger log = Logger.getLogger(ToolboxFileExporter.class); - private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-25 - all photo gallery assets.download.log"; + private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-28 - all logos and tags.download.log"; - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-25 - all photo gallery assets.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-28 - all logos and tags.csv"; private List rows; - private String baseDir = "D:\\ToolboxPhotoGallery"; + private String baseDir = "D:\\ToolboxLogosAndTags"; // Formatdatei generiert mit // bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ImportVerifier.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ImportVerifier.java new file mode 100644 index 0000000..82e5cc5 --- /dev/null +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ImportVerifier.java @@ -0,0 +1,95 @@ +package com.bayer.edam.excelimport; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; + +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.RollingFileAppender; + +import com.univocity.parsers.csv.CsvParser; +import com.univocity.parsers.csv.CsvParserSettings; + +/** + * Quick and dirty: Reads the files that made it into eDAM from a CSV list and determines their IDs from the import file. + * + * @author gexce + * + */ +public class ImportVerifier { + + private static Logger log = Logger.getLogger(ImportVerifier.class); + private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\ImportVerifier.log"; + + private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.csv"; + private static String migrationListingPath = "C:\\Temp\\Migration CS - PhotoGallery\\Frederic - Toolbox Migration DB export - COMPLETE.csv"; + + private CsvParserSettings parserSettings = new CsvParserSettings(); + private List importFile; + private List migrationListing; + + private HashSet assetIDs = new HashSet(); + private HashSet filenames = new HashSet(); + + public ImportVerifier() throws IOException { + parserSettings.getFormat().setLineSeparator("\r\n"); + parserSettings.getFormat().setDelimiter(';'); + parserSettings.getFormat().setCharToEscapeQuoteEscaping('"'); + parserSettings.setHeaderExtractionEnabled(false); + CsvParser parser = new CsvParser(parserSettings); + importFile = parser.parseAll(new File(importFilePath)); + migrationListing = parser.parseAll(new File(migrationListingPath)); + + long lineNo = 0; + for (String[] line : migrationListing) { + lineNo++; + String guid = line[0]; + String filename = line[1]; + + if (filenames.contains(filename)) { + log.warn("File '" + filename + "' has already been processed. Skipping line " + lineNo); + continue; + } + + filenames.add(filename); + boolean found = false; + for (String[] importLine : importFile) { + if (filename.equals(importLine[1])) { + assetIDs.add(Integer.parseInt(importLine[0])); + found = true; + } + } + + if (!found) { + log.error("No line for file " + filename); + } + } + + log.info("Number of asset IDs: " + assetIDs.size()); + + for (Integer assetID : assetIDs) { + log.info(assetID); + } + } + + public static void main(String[] args) { + try { + Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" + BasicConfigurator.configure(new ConsoleAppender(layout)); + BasicConfigurator.configure(new RollingFileAppender(layout, logfilePath, false)); + log.setLevel(Level.INFO); + + new ImportVerifier(); + + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java index f9e0e22..7dbd7f9 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java @@ -8,7 +8,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import org.apache.commons.io.FileUtils; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Layout; @@ -30,17 +29,12 @@ import com.univocity.parsers.csv.CsvWriterSettings; public class ToolboxImportFileGenerator { private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class); - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export.csv"; + private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export.log"; private static String licenseGuidsFilePath = "C:\\Temp\\Migration CS - SQL Server\\Release_GUIDs.csv"; - - // XXX - private static String migration2Path = "C:\\Temp\\Migration CS - SQL Server\\ToolboxMigration2-listing_incl_videos.txt"; - private List mig2Filenames; // XXX Spreadsheet needs be be named "Records" in Excel for the importer - private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases-IMPORT.csv"; - - private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.log"; + private static String importFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export-IMPORT.csv"; private static String uploadPath = "/Resources/Upload/ToolboxMigration"; private static final Map brandMap; @@ -1007,9 +1001,6 @@ public class ToolboxImportFileGenerator { } } - // XXX - mig2Filenames = FileUtils.readLines(new File(migration2Path), "UTF-8"); - generateAssetDecriptions(); writeImportFile(); } @@ -1076,14 +1067,6 @@ public class ToolboxImportFileGenerator { asset = new CropScienceAsset(assetId); asset.assetPath = uploadPath; - - // XXX - if (mig2Filenames.contains(line[1])) { - asset.assetPath = "/Resources/Upload/ToolboxMigration2"; - mig2++; - } - - asset.filename = line[1]; asset.byr_Title = line[2]; diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql b/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql index 132b056..63e171e 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql @@ -61,6 +61,9 @@ AND grp.GroupName != 'Administrator' -- Photo Gallery --AND a.FamilyID = 3 +-- Logos & Brand Tags +AND (a.AssetItemID = 46 OR a.AssetItemID = 67) + /* AND a.AssetID IN (24024,24014,24007,23933,23932,23830,23826,23805,23804,