tilman/toolbox-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ImportVerifier

Browse Source
This commit is contained in:
gexce
2017-08-28 14:23:05 +02:00
parent d244f60d1e
commit df8727f9b8
4 changed files with 104 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java
+3 -3
View File
@@ -19,12 +19,12 @@ import com.univocity.parsers.csv.CsvParserSettings;
public class ToolboxFileExporter {
private static Logger log = Logger.getLogger(ToolboxFileExporter.class);
private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-25 - all photo gallery assets.download.log";
private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-28 - all logos and tags.download.log";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-25 - all photo gallery assets.csv";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\export - 2017-08-28 - all logos and tags.csv";
private List<String[]> rows;
private String baseDir = "D:\\ToolboxPhotoGallery";
private String baseDir = "D:\\ToolboxLogosAndTags";
// Formatdatei generiert mit
// bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T
Toolbox Migration/src/com/bayer/edam/excelimport/ImportVerifier.java
+95
View File
@@ -0,0 +1,95 @@
package com.bayer.edam.excelimport;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
/**
* Quick and dirty: Reads the files that made it into eDAM from a CSV list and determines their IDs from the import file.
*
* @author gexce
*
*/
public class ImportVerifier {
private static Logger log = Logger.getLogger(ImportVerifier.class);
private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\ImportVerifier.log";
private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.csv";
private static String migrationListingPath = "C:\\Temp\\Migration CS - PhotoGallery\\Frederic - Toolbox Migration DB export - COMPLETE.csv";
private CsvParserSettings parserSettings = new CsvParserSettings();
private List<String[]> importFile;
private List<String[]> migrationListing;
private HashSet<Integer> assetIDs = new HashSet<Integer>();
private HashSet<String> filenames = new HashSet<String>();
public ImportVerifier() throws IOException {
parserSettings.getFormat().setLineSeparator("\r\n");
parserSettings.getFormat().setDelimiter(';');
parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
parserSettings.setHeaderExtractionEnabled(false);
CsvParser parser = new CsvParser(parserSettings);
importFile = parser.parseAll(new File(importFilePath));
migrationListing = parser.parseAll(new File(migrationListingPath));
long lineNo = 0;
for (String[] line : migrationListing) {
lineNo++;
String guid = line[0];
String filename = line[1];
if (filenames.contains(filename)) {
log.warn("File '" + filename + "' has already been processed. Skipping line " + lineNo);
continue;
}
filenames.add(filename);
boolean found = false;
for (String[] importLine : importFile) {
if (filename.equals(importLine[1])) {
assetIDs.add(Integer.parseInt(importLine[0]));
found = true;
}
}
if (!found) {
log.error("No line for file " + filename);
}
}
log.info("Number of asset IDs: " + assetIDs.size());
for (Integer assetID : assetIDs) {
log.info(assetID);
}
}
public static void main(String[] args) {
try {
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, logfilePath, false));
log.setLevel(Level.INFO);
new ImportVerifier();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java
+3 -20
View File
@@ -8,7 +8,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
@@ -30,17 +29,12 @@ import com.univocity.parsers.csv.CsvWriterSettings;
public class ToolboxImportFileGenerator {
private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class);
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv";
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export.csv";
private static String logfilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export.log";
private static String licenseGuidsFilePath = "C:\\Temp\\Migration CS - SQL Server\\Release_GUIDs.csv";
// XXX
private static String migration2Path = "C:\\Temp\\Migration CS - SQL Server\\ToolboxMigration2-listing_incl_videos.txt";
private List<String> mig2Filenames;
// XXX Spreadsheet needs be be named "Records" in Excel for the importer
private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases-IMPORT.csv";
private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.log";
private static String importFilePath = "C:\\Temp\\Migration CS - PhotoGallery\\all imported IDs - toolbox export-IMPORT.csv";
private static String uploadPath = "/Resources/Upload/ToolboxMigration";
private static final Map<String, String> brandMap;
@@ -1007,9 +1001,6 @@ public class ToolboxImportFileGenerator {
}
}
// XXX
mig2Filenames = FileUtils.readLines(new File(migration2Path), "UTF-8");
generateAssetDecriptions();
writeImportFile();
}
@@ -1076,14 +1067,6 @@ public class ToolboxImportFileGenerator {
asset = new CropScienceAsset(assetId);
asset.assetPath = uploadPath;
// XXX
if (mig2Filenames.contains(line[1])) {
asset.assetPath = "/Resources/Upload/ToolboxMigration2";
mig2++;
}
asset.filename = line[1];
asset.byr_Title = line[2];
Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql
+3
View File
@@ -61,6 +61,9 @@ AND grp.GroupName != 'Administrator'
-- Photo Gallery
--AND a.FamilyID = 3
-- Logos & Brand Tags
AND (a.AssetItemID = 46 OR a.AssetItemID = 67)
/*
AND a.AssetID IN (24024,24014,24007,23933,23932,23830,23826,23805,23804,