tilman/toolbox-migration
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CropAssetCopyTool

Browse Source
This commit is contained in:
gexce
2017-08-17 16:25:21 +02:00
parent 3c8afb360a
commit 21fa417bc8
1 changed files with 87 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java
+87 -17
View File
@@ -2,9 +2,14 @@ package com.bayer.edam.excelimport;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileExistsException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Layout;
@@ -15,6 +20,8 @@ import org.apache.log4j.RollingFileAppender;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
import com.univocity.parsers.csv.CsvWriter;
import com.univocity.parsers.csv.CsvWriterSettings;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
@@ -24,15 +31,15 @@ public class CropAssetCopyTool {
private static Logger log = Logger.getLogger(CropAssetCopyTool.class);
private static boolean dryRun = true;
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv";
private static String toolboxExportUpdateFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv";
private static String assetPath = "D:\\Toolbox\\Assets";
private static String videoPath = "D:\\Toolbox\\Movies\\";
private static File targetDir = new File("D:\\UploadAssets");
private static File targetDir = new File("D:\\UploadAssets\\");
private static String videoTargetPath = "D:\\UploadVideos";
private static File videoTargetDir = new File(videoTargetPath);
@@ -40,8 +47,16 @@ public class CropAssetCopyTool {
private CsvParserSettings parserSettings = new CsvParserSettings();
private List<String[]> inputRows;
private List<String[]> linesForDeletion = new ArrayList<String[]>();
HashSet<Integer> processedAssetIDs = new HashSet<Integer>();
public CropAssetCopyTool() throws IOException {
if (dryRun)
log.info("This is a dry run. No files are copied.");
readToolboxExport();
int lineNo = 0;
@@ -56,6 +71,12 @@ public class CropAssetCopyTool {
for (String[] line : inputRows) {
lineNo++;
Integer assetId = Integer.parseInt(line[0]);
if (processedAssetIDs.contains(assetId)) {
continue;
}
processedAssetIDs.add(assetId);
log.debug("Processing AssetID " + assetId + ", line " + lineNo);
String targetName = line[1];
@@ -68,9 +89,8 @@ public class CropAssetCopyTool {
if (file.isFile()) {
if (fileLength < 1) {
noContent++;
log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content");
// TODO update spreadsheet?
log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content. Removing file from list.");
linesForDeletion.add(line);
}
else if (expectedSize != fileLength) {
@@ -101,12 +121,18 @@ public class CropAssetCopyTool {
videosExtracted++;
}
else {
// XXX instead of copying the ZIP, extract only the video and link to the releases
log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP");
if (!dryRun)
if (!dryRun) {
File targetFile = FileUtils.getFile(videoTargetDir, videoFile.getName());
if (targetFile.exists())
throw new FileExistsException(targetFile);
FileUtils.copyFileToDirectory(videoFile, videoTargetDir);
}
// TODO update spreadsheet
line[1] = videoFile.getName();
videosZipped++;
}
@@ -115,16 +141,40 @@ public class CropAssetCopyTool {
} catch (ZipException e) {
zipCorrupt++;
log.error("Skipping AssetID " + assetId + " - could not open ZIP file " + videoFile.getAbsolutePath());
linesForDeletion.add(line);
}
}
else {
wrongSize++;
log.warn("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
log.debug("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
}
}
else {
// copy file
// if exists: add number and update spreadsheet
File targetFile = new File(targetDir + targetName);
if (targetFile.exists()) {
while (targetFile.exists()) {
String oldName = FilenameUtils.removeExtension(targetFile.getName());
String newName = oldName + "--002." + FilenameUtils.getExtension(oldName);
if (oldName.length() > 5 && oldName.charAt(oldName.length() - 4) == '-'
&& oldName.charAt(oldName.length() - 3) == '-'
&& StringUtils.isNumeric(oldName.substring(oldName.length() - 3))) {
int num = Integer.parseInt(oldName.substring(oldName.length() - 3));
num++;
newName = oldName.substring(0, oldName.length() - 3) + String.format("%03d", num) + "."
+ FilenameUtils.getExtension(oldName);
}
targetFile = new File(targetDir + newName);
}
log.info("(AssetID " + assetId + ") Target file '" + targetName + "' already exists. Copy file as '" + targetFile.getName() + "'");
line[1] = targetFile.getName();
}
log.debug("(AssetID " + assetId + ") Copy " + file.getName() + " to " + targetFile.getAbsolutePath());
if (!dryRun)
FileUtils.copyFile(file, targetFile);
}
}
else {
@@ -133,7 +183,13 @@ public class CropAssetCopyTool {
}
}
log.debug("Removing marked lines... " + inputRows.size());
inputRows.removeAll(linesForDeletion);
writeImportFile();
log.info("Lines processed: " + lineNo);
log.info("Assets processed: " + processedAssetIDs.size());
log.info("Skipped: File not found: " + noFile);
log.info("Skipped: No content: " + noContent);
log.info("Wrong size: " + wrongSize);
@@ -143,11 +199,6 @@ public class CropAssetCopyTool {
log.info(" Zipped: " + videosZipped);
log.info(" Corrupt: " + zipCorrupt);
// copy files by ID
// check for double filenames
// check for missing assets
// replace video assets
}
private void readToolboxExport() {
@@ -160,12 +211,31 @@ public class CropAssetCopyTool {
inputRows = parser.parseAll(new File(toolboxExportFilePath));
}
private void writeImportFile() {
CsvWriterSettings writerSettings = new CsvWriterSettings();
writerSettings.setFormat(parserSettings.getFormat());
List<Object[]> csvOutputList = new ArrayList<Object[]>();
for (String[] line : inputRows) {
csvOutputList.add(line);
}
CsvWriter writer = new CsvWriter(new File(toolboxExportUpdateFilePath), writerSettings);
writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId",
"fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
"fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished",
"fields|byr_cs_warehouseAvailable");
writer.writeRowsAndClose(csvOutputList);
}
public static void main(String[] args) {
try {
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
BasicConfigurator.configure(new ConsoleAppender(layout));
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true));
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", false));
log.setLevel(Level.INFO);
new CropAssetCopyTool();