diff --git a/Toolbox Migration/pom.xml b/Toolbox Migration/pom.xml index db95963..c267cb4 100644 --- a/Toolbox Migration/pom.xml +++ b/Toolbox Migration/pom.xml @@ -39,6 +39,11 @@ selenium-firefox-driver 3.4.0 + + net.lingala.zip4j + zip4j + 1.3.2 + diff --git a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java index 1613693..82149fd 100644 --- a/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java +++ b/Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java @@ -22,12 +22,12 @@ public class ToolboxFileExporter { private List rows; - private String baseDir = "F:\\Toolbox\\Assets"; + private String baseDir = "D:\\Toolbox\\Assets"; // Formatdatei generiert mit // bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T // Geben Sie den Dateispeichertyp des Felds OriginalImage ein [varbinary(max)]: x - // Geben Sie die Präfixlänge des OriginalImage-Felds ein [8]: 0 + // Geben Sie die Pr�fixl�nge des OriginalImage-Felds ein [8]: 0 // Geben Sie das Feldabschlusszeichen ein [none]: // Abruf: @@ -35,7 +35,7 @@ public class ToolboxFileExporter { private String command1 = "bcp \"SELECT OriginalImage FROM AssetImages WHERE AssetID="; private String command2 = "\" queryout \""; - private String command3 = "\" -S MOVNCG -d Bayertoolbox_prod -T -f \"F:\\Toolbox\\bcp.fmt\" -a 16384"; + private String command3 = "\" -S MOVNCG -d Bayertoolbox_prod -T -f \"D:\\Toolbox\\bcp.fmt\" -a 16384"; private File workingDir = new File(baseDir); @@ -43,7 +43,7 @@ public class ToolboxFileExporter { readList(); for (String[] row : rows) { - getImage(row[1], row[3]); + getImage(row[0], row[1]); } } @@ -75,15 +75,15 @@ public class ToolboxFileExporter { CsvParser parser = new CsvParser(settings); - rows = parser.parseAll(new File("F:\\Toolbox\\AssetImages.csv")); + rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv")); } public static void main(String[] args) { try { - Layout layout = new PatternLayout("%p %m%n"); // "%d{ISO8601} - %p %m%n" + Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" BasicConfigurator.configure(new ConsoleAppender(layout)); - BasicConfigurator.configure(new RollingFileAppender(layout, "F:\\Toolbox\\download.log", true)); + BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\Toolbox\\download.log", true)); log.setLevel(Level.DEBUG); long startTime = System.currentTimeMillis(); diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java b/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java new file mode 100644 index 0000000..777aa3e --- /dev/null +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java @@ -0,0 +1,155 @@ +package com.bayer.edam.excelimport; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.RollingFileAppender; + +import com.univocity.parsers.csv.CsvParser; +import com.univocity.parsers.csv.CsvParserSettings; + +import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.model.FileHeader; + +public class CropAssetCopyTool { + + private static Logger log = Logger.getLogger(CropAssetCopyTool.class); + + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv"; + private static String assetPath = "D:\\Toolbox\\Assets"; + private static String videoPath = "D:\\Toolbox\\Movies\\"; + + private static File targetDir = new File("D:\\UploadAssets"); + private static String videoTargetPath = "D:\\UploadVideos"; + private static File videoTargetDir = new File(videoTargetPath); + + + + private CsvParserSettings parserSettings = new CsvParserSettings(); + private List inputRows; + + public CropAssetCopyTool() throws ZipException, IOException { + readToolboxExport(); + + int lineNo = 0; + int noContent = 0; + int wrongSize = 0; + int video = 0; + int videosExtracted = 0; + int videosZipped = 0; + + for (String[] line : inputRows) { + lineNo++; + Integer assetId = Integer.parseInt(line[0]); + log.debug("Processing AssetID " + assetId + ", line " + lineNo); + + String targetName = line[1]; + long expectedSize = Long.parseLong(line[15]); + String filePath = assetPath + "\\" + assetId; + + File file = new File(filePath); + long fileLength = file.length(); + + if (file.isFile()) { + if (fileLength < 1) { + noContent++; + log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content"); + } + else if (expectedSize != fileLength) { + + File videoFile = new File(videoPath + assetId + ".zip"); + if (videoFile.exists()) { + ZipFile zipFile = new ZipFile(videoFile.getAbsolutePath()); + + @SuppressWarnings("rawtypes") + List fileList = zipFile.getFileHeaders(); + + if (fileList.size() == 1) { + log.info("Extracting " + videoFile.getAbsolutePath()); +// zipFile.extractAll(videoTargetPath); +// +// FileHeader header = (FileHeader) fileList.get(0); +// File srcFile = new File(videoTargetPath + "\\" + header.getFileName()); +// File destFile = new File(videoTargetPath + "\\" + targetName); +// log.info("AssetID " + assetId + ": Moving " + srcFile.getAbsolutePath() + " to " + destFile.getAbsolutePath()); +// +// FileUtils.moveFile(srcFile, destFile); + + videosExtracted++; + } + else { + log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP"); +// FileUtils.copyFileToDirectory(videoFile, videoTargetDir); + + // TODO update spreadsheet + + videosZipped++; + } + + video++; + } + else { + wrongSize++; + log.warn("Wrong file size for " + assetId + ": " + expectedSize + " != " + fileLength + " (" + targetName + ")"); + } + } + else { + // copy file + // if exists: add number and update spreadsheet + } + } + else { + log.error("'" + filePath + "' is not a file"); + } + } + + log.info("Lines processed: " + lineNo); + log.info("Skipped: No content: " + noContent); + log.info("Wrong size: " + wrongSize); + + log.info("Videos: " + video); + log.info(" Extracted: " + videosExtracted); + log.info(" Zipped: " + videosZipped); + + + // copy files by ID + // check for double filenames + // check for missing assets + // replace video assets + } + + private void readToolboxExport() { + parserSettings.getFormat().setLineSeparator("\r\n"); + parserSettings.getFormat().setDelimiter(';'); + parserSettings.getFormat().setCharToEscapeQuoteEscaping('"'); + parserSettings.setHeaderExtractionEnabled(false); + + CsvParser parser = new CsvParser(parserSettings); + + inputRows = parser.parseAll(new File(toolboxExportFilePath)); + } + + public static void main(String[] args) { + try { + Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" + BasicConfigurator.configure(new ConsoleAppender(layout)); + BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true)); + log.setLevel(Level.INFO); + + new CropAssetCopyTool(); + + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java index e82a764..744db69 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java @@ -1178,8 +1178,11 @@ public class ToolboxImportFileGenerator { for (CropScienceAsset asset : assets.values()) { String brands = ""; - for (String brand : asset.brands) + for (String brand : asset.brands) { + if (brands.length() > 0) + brands += "\n"; brands += brand; + } for (String indication : asset.indications) asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication;