From 21fa417bc81eeaf8b3ef6498d257704dee4157f4 Mon Sep 17 00:00:00 2001 From: gexce Date: Thu, 17 Aug 2017 16:25:21 +0200 Subject: [PATCH] CropAssetCopyTool --- .../edam/excelimport/CropAssetCopyTool.java | 104 +++++++++++++++--- 1 file changed, 87 insertions(+), 17 deletions(-) diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java b/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java index ae06b33..ae3389d 100644 --- a/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java @@ -2,9 +2,14 @@ package com.bayer.edam.excelimport; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import org.apache.commons.io.FileExistsException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Layout; @@ -15,6 +20,8 @@ import org.apache.log4j.RollingFileAppender; import com.univocity.parsers.csv.CsvParser; import com.univocity.parsers.csv.CsvParserSettings; +import com.univocity.parsers.csv.CsvWriter; +import com.univocity.parsers.csv.CsvWriterSettings; import net.lingala.zip4j.core.ZipFile; import net.lingala.zip4j.exception.ZipException; @@ -24,15 +31,15 @@ public class CropAssetCopyTool { private static Logger log = Logger.getLogger(CropAssetCopyTool.class); - private static boolean dryRun = true; private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv"; + private static String toolboxExportUpdateFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv"; private static String assetPath = "D:\\Toolbox\\Assets"; private static String videoPath = "D:\\Toolbox\\Movies\\"; - private static File targetDir = new File("D:\\UploadAssets"); + private static File targetDir = new File("D:\\UploadAssets\\"); private static String videoTargetPath = "D:\\UploadVideos"; private static File videoTargetDir = new File(videoTargetPath); @@ -40,8 +47,16 @@ public class CropAssetCopyTool { private CsvParserSettings parserSettings = new CsvParserSettings(); private List inputRows; + private List linesForDeletion = new ArrayList(); + + HashSet processedAssetIDs = new HashSet(); public CropAssetCopyTool() throws IOException { + + if (dryRun) + log.info("This is a dry run. No files are copied."); + + readToolboxExport(); int lineNo = 0; @@ -56,6 +71,12 @@ public class CropAssetCopyTool { for (String[] line : inputRows) { lineNo++; Integer assetId = Integer.parseInt(line[0]); + + if (processedAssetIDs.contains(assetId)) { + continue; + } + + processedAssetIDs.add(assetId); log.debug("Processing AssetID " + assetId + ", line " + lineNo); String targetName = line[1]; @@ -68,9 +89,8 @@ public class CropAssetCopyTool { if (file.isFile()) { if (fileLength < 1) { noContent++; - log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content"); - - // TODO update spreadsheet? + log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content. Removing file from list."); + linesForDeletion.add(line); } else if (expectedSize != fileLength) { @@ -101,12 +121,18 @@ public class CropAssetCopyTool { videosExtracted++; } else { + // XXX instead of copying the ZIP, extract only the video and link to the releases + log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP"); - if (!dryRun) + if (!dryRun) { + File targetFile = FileUtils.getFile(videoTargetDir, videoFile.getName()); + if (targetFile.exists()) + throw new FileExistsException(targetFile); + FileUtils.copyFileToDirectory(videoFile, videoTargetDir); + } - // TODO update spreadsheet - + line[1] = videoFile.getName(); videosZipped++; } @@ -115,16 +141,40 @@ public class CropAssetCopyTool { } catch (ZipException e) { zipCorrupt++; log.error("Skipping AssetID " + assetId + " - could not open ZIP file " + videoFile.getAbsolutePath()); + linesForDeletion.add(line); } } else { wrongSize++; - log.warn("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")"); + log.debug("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")"); } } else { - // copy file - // if exists: add number and update spreadsheet + File targetFile = new File(targetDir + targetName); + + if (targetFile.exists()) { + while (targetFile.exists()) { + String oldName = FilenameUtils.removeExtension(targetFile.getName()); + String newName = oldName + "--002." + FilenameUtils.getExtension(oldName); + if (oldName.length() > 5 && oldName.charAt(oldName.length() - 4) == '-' + && oldName.charAt(oldName.length() - 3) == '-' + && StringUtils.isNumeric(oldName.substring(oldName.length() - 3))) { + int num = Integer.parseInt(oldName.substring(oldName.length() - 3)); + num++; + newName = oldName.substring(0, oldName.length() - 3) + String.format("%03d", num) + "." + + FilenameUtils.getExtension(oldName); + } + targetFile = new File(targetDir + newName); + } + + log.info("(AssetID " + assetId + ") Target file '" + targetName + "' already exists. Copy file as '" + targetFile.getName() + "'"); + line[1] = targetFile.getName(); + } + + log.debug("(AssetID " + assetId + ") Copy " + file.getName() + " to " + targetFile.getAbsolutePath()); + + if (!dryRun) + FileUtils.copyFile(file, targetFile); } } else { @@ -133,7 +183,13 @@ public class CropAssetCopyTool { } } + log.debug("Removing marked lines... " + inputRows.size()); + inputRows.removeAll(linesForDeletion); + + writeImportFile(); + log.info("Lines processed: " + lineNo); + log.info("Assets processed: " + processedAssetIDs.size()); log.info("Skipped: File not found: " + noFile); log.info("Skipped: No content: " + noContent); log.info("Wrong size: " + wrongSize); @@ -143,11 +199,6 @@ public class CropAssetCopyTool { log.info(" Zipped: " + videosZipped); log.info(" Corrupt: " + zipCorrupt); - - // copy files by ID - // check for double filenames - // check for missing assets - // replace video assets } private void readToolboxExport() { @@ -160,12 +211,31 @@ public class CropAssetCopyTool { inputRows = parser.parseAll(new File(toolboxExportFilePath)); } + + private void writeImportFile() { + CsvWriterSettings writerSettings = new CsvWriterSettings(); + writerSettings.setFormat(parserSettings.getFormat()); + + List csvOutputList = new ArrayList(); + for (String[] line : inputRows) { + csvOutputList.add(line); + } + + CsvWriter writer = new CsvWriter(new File(toolboxExportUpdateFilePath), writerSettings); + writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId", + "fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand", + "fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl", + "fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished", + "fields|byr_cs_warehouseAvailable"); + writer.writeRowsAndClose(csvOutputList); + } public static void main(String[] args) { + try { Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n" BasicConfigurator.configure(new ConsoleAppender(layout)); - BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true)); + BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", false)); log.setLevel(Level.INFO); new CropAssetCopyTool();