CropAssetCopyTool
This commit is contained in:
gexce
@@ -2,9 +2,14 @@ package com.bayer.edam.excelimport;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.FileExistsException;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.log4j.BasicConfigurator;
|
||||
import org.apache.log4j.ConsoleAppender;
|
||||
import org.apache.log4j.Layout;
|
||||
@@ -15,6 +20,8 @@ import org.apache.log4j.RollingFileAppender;
|
||||
|
||||
import com.univocity.parsers.csv.CsvParser;
|
||||
import com.univocity.parsers.csv.CsvParserSettings;
|
||||
import com.univocity.parsers.csv.CsvWriter;
|
||||
import com.univocity.parsers.csv.CsvWriterSettings;
|
||||
|
||||
import net.lingala.zip4j.core.ZipFile;
|
||||
import net.lingala.zip4j.exception.ZipException;
|
||||
@@ -24,15 +31,15 @@ public class CropAssetCopyTool {
|
||||
|
||||
private static Logger log = Logger.getLogger(CropAssetCopyTool.class);
|
||||
|
||||
|
||||
private static boolean dryRun = true;
|
||||
|
||||
|
||||
private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv";
|
||||
private static String toolboxExportUpdateFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv";
|
||||
private static String assetPath = "D:\\Toolbox\\Assets";
|
||||
private static String videoPath = "D:\\Toolbox\\Movies\\";
|
||||
|
||||
private static File targetDir = new File("D:\\UploadAssets");
|
||||
private static File targetDir = new File("D:\\UploadAssets\\");
|
||||
private static String videoTargetPath = "D:\\UploadVideos";
|
||||
private static File videoTargetDir = new File(videoTargetPath);
|
||||
|
||||
@@ -40,8 +47,16 @@ public class CropAssetCopyTool {
|
||||
|
||||
private CsvParserSettings parserSettings = new CsvParserSettings();
|
||||
private List<String[]> inputRows;
|
||||
private List<String[]> linesForDeletion = new ArrayList<String[]>();
|
||||
|
||||
HashSet<Integer> processedAssetIDs = new HashSet<Integer>();
|
||||
|
||||
public CropAssetCopyTool() throws IOException {
|
||||
|
||||
if (dryRun)
|
||||
log.info("This is a dry run. No files are copied.");
|
||||
|
||||
|
||||
readToolboxExport();
|
||||
|
||||
int lineNo = 0;
|
||||
@@ -56,6 +71,12 @@ public class CropAssetCopyTool {
|
||||
for (String[] line : inputRows) {
|
||||
lineNo++;
|
||||
Integer assetId = Integer.parseInt(line[0]);
|
||||
|
||||
if (processedAssetIDs.contains(assetId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
processedAssetIDs.add(assetId);
|
||||
log.debug("Processing AssetID " + assetId + ", line " + lineNo);
|
||||
|
||||
String targetName = line[1];
|
||||
@@ -68,9 +89,8 @@ public class CropAssetCopyTool {
|
||||
if (file.isFile()) {
|
||||
if (fileLength < 1) {
|
||||
noContent++;
|
||||
log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content");
|
||||
|
||||
// TODO update spreadsheet?
|
||||
log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content. Removing file from list.");
|
||||
linesForDeletion.add(line);
|
||||
}
|
||||
else if (expectedSize != fileLength) {
|
||||
|
||||
@@ -101,12 +121,18 @@ public class CropAssetCopyTool {
|
||||
videosExtracted++;
|
||||
}
|
||||
else {
|
||||
// XXX instead of copying the ZIP, extract only the video and link to the releases
|
||||
|
||||
log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP");
|
||||
if (!dryRun)
|
||||
if (!dryRun) {
|
||||
File targetFile = FileUtils.getFile(videoTargetDir, videoFile.getName());
|
||||
if (targetFile.exists())
|
||||
throw new FileExistsException(targetFile);
|
||||
|
||||
FileUtils.copyFileToDirectory(videoFile, videoTargetDir);
|
||||
}
|
||||
|
||||
// TODO update spreadsheet
|
||||
|
||||
line[1] = videoFile.getName();
|
||||
videosZipped++;
|
||||
}
|
||||
|
||||
@@ -115,16 +141,40 @@ public class CropAssetCopyTool {
|
||||
} catch (ZipException e) {
|
||||
zipCorrupt++;
|
||||
log.error("Skipping AssetID " + assetId + " - could not open ZIP file " + videoFile.getAbsolutePath());
|
||||
linesForDeletion.add(line);
|
||||
}
|
||||
}
|
||||
else {
|
||||
wrongSize++;
|
||||
log.warn("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
|
||||
log.debug("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
|
||||
}
|
||||
}
|
||||
else {
|
||||
// copy file
|
||||
// if exists: add number and update spreadsheet
|
||||
File targetFile = new File(targetDir + targetName);
|
||||
|
||||
if (targetFile.exists()) {
|
||||
while (targetFile.exists()) {
|
||||
String oldName = FilenameUtils.removeExtension(targetFile.getName());
|
||||
String newName = oldName + "--002." + FilenameUtils.getExtension(oldName);
|
||||
if (oldName.length() > 5 && oldName.charAt(oldName.length() - 4) == '-'
|
||||
&& oldName.charAt(oldName.length() - 3) == '-'
|
||||
&& StringUtils.isNumeric(oldName.substring(oldName.length() - 3))) {
|
||||
int num = Integer.parseInt(oldName.substring(oldName.length() - 3));
|
||||
num++;
|
||||
newName = oldName.substring(0, oldName.length() - 3) + String.format("%03d", num) + "."
|
||||
+ FilenameUtils.getExtension(oldName);
|
||||
}
|
||||
targetFile = new File(targetDir + newName);
|
||||
}
|
||||
|
||||
log.info("(AssetID " + assetId + ") Target file '" + targetName + "' already exists. Copy file as '" + targetFile.getName() + "'");
|
||||
line[1] = targetFile.getName();
|
||||
}
|
||||
|
||||
log.debug("(AssetID " + assetId + ") Copy " + file.getName() + " to " + targetFile.getAbsolutePath());
|
||||
|
||||
if (!dryRun)
|
||||
FileUtils.copyFile(file, targetFile);
|
||||
}
|
||||
}
|
||||
else {
|
||||
@@ -133,7 +183,13 @@ public class CropAssetCopyTool {
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("Removing marked lines... " + inputRows.size());
|
||||
inputRows.removeAll(linesForDeletion);
|
||||
|
||||
writeImportFile();
|
||||
|
||||
log.info("Lines processed: " + lineNo);
|
||||
log.info("Assets processed: " + processedAssetIDs.size());
|
||||
log.info("Skipped: File not found: " + noFile);
|
||||
log.info("Skipped: No content: " + noContent);
|
||||
log.info("Wrong size: " + wrongSize);
|
||||
@@ -143,11 +199,6 @@ public class CropAssetCopyTool {
|
||||
log.info(" Zipped: " + videosZipped);
|
||||
log.info(" Corrupt: " + zipCorrupt);
|
||||
|
||||
|
||||
// copy files by ID
|
||||
// check for double filenames
|
||||
// check for missing assets
|
||||
// replace video assets
|
||||
}
|
||||
|
||||
private void readToolboxExport() {
|
||||
@@ -161,11 +212,30 @@ public class CropAssetCopyTool {
|
||||
inputRows = parser.parseAll(new File(toolboxExportFilePath));
|
||||
}
|
||||
|
||||
private void writeImportFile() {
|
||||
CsvWriterSettings writerSettings = new CsvWriterSettings();
|
||||
writerSettings.setFormat(parserSettings.getFormat());
|
||||
|
||||
List<Object[]> csvOutputList = new ArrayList<Object[]>();
|
||||
for (String[] line : inputRows) {
|
||||
csvOutputList.add(line);
|
||||
}
|
||||
|
||||
CsvWriter writer = new CsvWriter(new File(toolboxExportUpdateFilePath), writerSettings);
|
||||
writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId",
|
||||
"fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
|
||||
"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
|
||||
"fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished",
|
||||
"fields|byr_cs_warehouseAvailable");
|
||||
writer.writeRowsAndClose(csvOutputList);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
try {
|
||||
Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
|
||||
BasicConfigurator.configure(new ConsoleAppender(layout));
|
||||
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true));
|
||||
BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", false));
|
||||
log.setLevel(Level.INFO);
|
||||
|
||||
new CropAssetCopyTool();
|
||||
|
||||
Reference in New Issue
Block a user