tilman/toolbox-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CropAssetCopyTool

Browse Source
This commit is contained in:
gexce
2017-08-17 13:01:14 +02:00
parent 3c18cb1418
commit 31e365af05
4 changed files with 171 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Toolbox Migration/pom.xml
+5
View File
@@ -39,6 +39,11 @@
<artifactId>selenium-firefox-driver</artifactId>
<version>3.4.0</version>
</dependency>
<dependency>
<groupId>net.lingala.zip4j</groupId>
<artifactId>zip4j</artifactId>
<version>1.3.2</version>
</dependency>
</dependencies>
<build>
Toolbox Migration/src/com/bayer/edam/ToolboxFileExporter.java
+7 -7
View File
@@ -22,12 +22,12 @@ public class ToolboxFileExporter {
private List<String[]> rows;
private String baseDir = "F:\\Toolbox\\Assets";
private String baseDir = "D:\\Toolbox\\Assets";
// Formatdatei generiert mit
// bcp "SELECT OriginalImage FROM AssetImages WHERE AssetID=586" queryout Image.jpg -S MOVNCG -d Bayertoolbox_prod -T
// Geben Sie den Dateispeichertyp des Felds OriginalImage ein [varbinary(max)]: x
// Geben Sie die Präfixlänge des OriginalImage-Felds ein [8]: 0
// Geben Sie die Pr�fixl�nge des OriginalImage-Felds ein [8]: 0
// Geben Sie das Feldabschlusszeichen ein [none]:
// Abruf:
@@ -35,7 +35,7 @@ public class ToolboxFileExporter {
private String command1 = "bcp \"SELECT OriginalImage FROM AssetImages WHERE AssetID=";
private String command2 = "\" queryout \"";
private String command3 = "\" -S MOVNCG -d Bayertoolbox_prod -T -f \"F:\\Toolbox\\bcp.fmt\" -a 16384";
private String command3 = "\" -S MOVNCG -d Bayertoolbox_prod -T -f \"D:\\Toolbox\\bcp.fmt\" -a 16384";
private File workingDir = new File(baseDir);
@@ -43,7 +43,7 @@ public class ToolboxFileExporter {
readList();
for (String[] row : rows) {
getImage(row[1], row[3]);
getImage(row[0], row[1]);
}
}
@@ -75,15 +75,15 @@ public class ToolboxFileExporter {
CsvParser parser = new CsvParser(settings);
rows = parser.parseAll(new File("F:\\Toolbox\\AssetImages.csv"));
rows = parser.parseAll(new File("C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv"));
}
public static void main(String[] args) {
try {
Layout layout = new PatternLayout("%p %m%n"); // "%d{ISO8601} - %p %m%n"
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, "F:\\Toolbox\\download.log", true));
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\Toolbox\\download.log", true));
log.setLevel(Level.DEBUG);
long startTime = System.currentTimeMillis();
Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java
+155
View File
@@ -0,0 +1,155 @@
package com.bayer.edam.excelimport;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import net.lingala.zip4j.model.FileHeader;
public class CropAssetCopyTool {
private static Logger log = Logger.getLogger(CropAssetCopyTool.class);
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv";
private static String assetPath = "D:\\Toolbox\\Assets";
private static String videoPath = "D:\\Toolbox\\Movies\\";
private static File targetDir = new File("D:\\UploadAssets");
private static String videoTargetPath = "D:\\UploadVideos";
private static File videoTargetDir = new File(videoTargetPath);
private CsvParserSettings parserSettings = new CsvParserSettings();
private List<String[]> inputRows;
public CropAssetCopyTool() throws ZipException, IOException {
readToolboxExport();
int lineNo = 0;
int noContent = 0;
int wrongSize = 0;
int video = 0;
int videosExtracted = 0;
int videosZipped = 0;
for (String[] line : inputRows) {
lineNo++;
Integer assetId = Integer.parseInt(line[0]);
log.debug("Processing AssetID " + assetId + ", line " + lineNo);
String targetName = line[1];
long expectedSize = Long.parseLong(line[15]);
String filePath = assetPath + "\\" + assetId;
File file = new File(filePath);
long fileLength = file.length();
if (file.isFile()) {
if (fileLength < 1) {
noContent++;
log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content");
}
else if (expectedSize != fileLength) {
File videoFile = new File(videoPath + assetId + ".zip");
if (videoFile.exists()) {
ZipFile zipFile = new ZipFile(videoFile.getAbsolutePath());
@SuppressWarnings("rawtypes")
List fileList = zipFile.getFileHeaders();
if (fileList.size() == 1) {
log.info("Extracting " + videoFile.getAbsolutePath());
// zipFile.extractAll(videoTargetPath);
//
// FileHeader header = (FileHeader) fileList.get(0);
// File srcFile = new File(videoTargetPath + "\\" + header.getFileName());
// File destFile = new File(videoTargetPath + "\\" + targetName);
// log.info("AssetID " + assetId + ": Moving " + srcFile.getAbsolutePath() + " to " + destFile.getAbsolutePath());
//
// FileUtils.moveFile(srcFile, destFile);
videosExtracted++;
}
else {
log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP");
// FileUtils.copyFileToDirectory(videoFile, videoTargetDir);
// TODO update spreadsheet
videosZipped++;
}
video++;
}
else {
wrongSize++;
log.warn("Wrong file size for " + assetId + ": " + expectedSize + " != " + fileLength + " (" + targetName + ")");
}
}
else {
// copy file
// if exists: add number and update spreadsheet
}
}
else {
log.error("'" + filePath + "' is not a file");
}
}
log.info("Lines processed: " + lineNo);
log.info("Skipped: No content: " + noContent);
log.info("Wrong size: " + wrongSize);
log.info("Videos: " + video);
log.info(" Extracted: " + videosExtracted);
log.info(" Zipped: " + videosZipped);
// copy files by ID
// check for double filenames
// check for missing assets
// replace video assets
}
private void readToolboxExport() {
parserSettings.getFormat().setLineSeparator("\r\n");
parserSettings.getFormat().setDelimiter(';');
parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
parserSettings.setHeaderExtractionEnabled(false);
CsvParser parser = new CsvParser(parserSettings);
inputRows = parser.parseAll(new File(toolboxExportFilePath));
}
public static void main(String[] args) {
try {
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true));
log.setLevel(Level.INFO);
new CropAssetCopyTool();
} catch (Exception e) {
e.printStackTrace();
}
}
}
Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java
+4 -1
View File
@@ -1178,8 +1178,11 @@ public class ToolboxImportFileGenerator {
for (CropScienceAsset asset : assets.values()) {
String brands = "";
for (String brand : asset.brands)
for (String brand : asset.brands) {
if (brands.length() > 0)
brands += "\n";
brands += brand;
}
for (String indication : asset.indications)
asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication;