CropAssetCopyTool

Toolbox Migration/src/com/bayer/edam/excelimport/CropAssetCopyTool.java

@@ -2,9 +2,14 @@ package com.bayer.edam.excelimport;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 
+import org.apache.commons.io.FileExistsException;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.Layout;
@@ -15,6 +20,8 @@ import org.apache.log4j.RollingFileAppender;
 
 import com.univocity.parsers.csv.CsvParser;
 import com.univocity.parsers.csv.CsvParserSettings;
+import com.univocity.parsers.csv.CsvWriter;
+import com.univocity.parsers.csv.CsvWriterSettings;
 
 import net.lingala.zip4j.core.ZipFile;
 import net.lingala.zip4j.exception.ZipException;
@@ -24,15 +31,15 @@ public class CropAssetCopyTool {
 	
 	private static Logger log = Logger.getLogger(CropAssetCopyTool.class);
 	
-	
 	private static boolean dryRun = true;
 	
 	
 	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv";
+	private static String toolboxExportUpdateFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update.csv";
 	private static String assetPath = "D:\\Toolbox\\Assets";
 	private static String videoPath = "D:\\Toolbox\\Movies\\";
 	
-	private static File targetDir = new File("D:\\UploadAssets");
+	private static File targetDir = new File("D:\\UploadAssets\\");
 	private static String videoTargetPath = "D:\\UploadVideos";
 	private static File videoTargetDir = new File(videoTargetPath);
 	
@@ -40,8 +47,16 @@ public class CropAssetCopyTool {
 	
 	private CsvParserSettings parserSettings = new CsvParserSettings();
 	private List<String[]> inputRows;
+	private List<String[]> linesForDeletion = new ArrayList<String[]>();
+	
+	HashSet<Integer> processedAssetIDs = new HashSet<Integer>();
 	
 	public CropAssetCopyTool() throws IOException {
+		
+		if (dryRun)
+			log.info("This is a dry run. No files are copied.");
+		
+		
 		readToolboxExport();
 		
 		int lineNo = 0;
@@ -56,6 +71,12 @@ public class CropAssetCopyTool {
 		for (String[] line : inputRows) {
 			lineNo++;
 			Integer assetId = Integer.parseInt(line[0]);
+			
+			if (processedAssetIDs.contains(assetId)) {
+				continue;
+			}
+			
+			processedAssetIDs.add(assetId);
 			log.debug("Processing AssetID " + assetId + ", line " + lineNo);
 			
 			String targetName = line[1];
@@ -68,9 +89,8 @@ public class CropAssetCopyTool {
 			if (file.isFile()) {
 				if (fileLength < 1) {
 					noContent++;
-					log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content");
+					log.warn("Skipping '" + filePath + "' (AssetID " + assetId + "), no content. Removing file from list.");
-					
+					linesForDeletion.add(line);
-					// TODO update spreadsheet?
 				}
 				else if (expectedSize != fileLength) {
 					
@@ -101,12 +121,18 @@ public class CropAssetCopyTool {
 								videosExtracted++;
 							}
 							else {
+								// XXX instead of copying the ZIP, extract only the video and link to the releases
+								
 								log.info("Copy " + videoFile.getAbsolutePath() + " as ZIP");
-								if (!dryRun)
+								if (!dryRun) {
+									File targetFile = FileUtils.getFile(videoTargetDir, videoFile.getName());
+									if (targetFile.exists())
+										throw new FileExistsException(targetFile);
+									
 									FileUtils.copyFileToDirectory(videoFile, videoTargetDir);
+								}
 								
-								// TODO update spreadsheet
+								line[1] = videoFile.getName();
-								
 								videosZipped++;
 							}
 							
@@ -115,16 +141,40 @@ public class CropAssetCopyTool {
 						} catch (ZipException e) {
 							zipCorrupt++;
 							log.error("Skipping AssetID " + assetId + " - could not open ZIP file " + videoFile.getAbsolutePath());
+							linesForDeletion.add(line);
 						}
 					}
 					else {
 						wrongSize++;
-						log.warn("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
+						log.debug("Wrong file size for " + assetId + ": expected " + expectedSize + ", found " + fileLength + " (" + targetName + ")");
 					}
 				}
 				else {
-					// copy file
+					File targetFile = new File(targetDir + targetName);
-					// if exists: add number and update spreadsheet
+					
+					if (targetFile.exists()) {
+						while (targetFile.exists()) {
+							String oldName = FilenameUtils.removeExtension(targetFile.getName());
+							String newName = oldName + "--002." + FilenameUtils.getExtension(oldName);
+							if (oldName.length() > 5 && oldName.charAt(oldName.length() - 4) == '-'
+									&& oldName.charAt(oldName.length() - 3) == '-'
+									&& StringUtils.isNumeric(oldName.substring(oldName.length() - 3))) {
+								int num = Integer.parseInt(oldName.substring(oldName.length() - 3));
+								num++;
+								newName = oldName.substring(0, oldName.length() - 3) + String.format("%03d", num) + "."
+										+ FilenameUtils.getExtension(oldName);
+							}
+							targetFile = new File(targetDir + newName);
+						}
+						
+						log.info("(AssetID " + assetId + ") Target file '" + targetName + "' already exists. Copy file as  '" + targetFile.getName() + "'");
+						line[1] = targetFile.getName();
+					}
+					
+					log.debug("(AssetID " + assetId + ") Copy " + file.getName() + " to " + targetFile.getAbsolutePath());
+					
+					if (!dryRun)
+						FileUtils.copyFile(file, targetFile);
 				}
 			}
 			else {
@@ -133,7 +183,13 @@ public class CropAssetCopyTool {
 			}
 		}
 		
+		log.debug("Removing marked lines... " + inputRows.size());
+		inputRows.removeAll(linesForDeletion);
+		
+		writeImportFile();
+		
 		log.info("Lines processed: " + lineNo);
+		log.info("Assets processed: " + processedAssetIDs.size());
 		log.info("Skipped: File not found: " + noFile);
 		log.info("Skipped: No content: " + noContent);
 		log.info("Wrong size: " + wrongSize);
@@ -143,11 +199,6 @@ public class CropAssetCopyTool {
 		log.info("  Zipped: " + videosZipped);
 		log.info("  Corrupt: " + zipCorrupt);
 		
-		
-		// copy files by ID
-		//   check for double filenames
-		//   check for missing assets
-		//   replace video assets
 	}
 	
 	private void readToolboxExport() {
@@ -161,11 +212,30 @@ public class CropAssetCopyTool {
 		inputRows = parser.parseAll(new File(toolboxExportFilePath));
 	}
 
+	private void writeImportFile() {
+		CsvWriterSettings writerSettings = new CsvWriterSettings();
+		writerSettings.setFormat(parserSettings.getFormat());
+		
+		List<Object[]> csvOutputList = new ArrayList<Object[]>();
+		for (String[] line : inputRows) {
+			csvOutputList.add(line);
+		}
+
+		CsvWriter writer = new CsvWriter(new File(toolboxExportUpdateFilePath), writerSettings);
+		writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId",
+				"fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
+				"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
+				"fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished",
+				"fields|byr_cs_warehouseAvailable");
+		writer.writeRowsAndClose(csvOutputList);
+	}
+	
 	public static void main(String[] args) {
+		
 		try {
 			Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
 			BasicConfigurator.configure(new ConsoleAppender(layout));
-			BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", true));
+			BasicConfigurator.configure(new RollingFileAppender(layout, "D:\\CropAssetCopyTool.log", false));
 			log.setLevel(Level.INFO);
 
 			new CropAssetCopyTool();