adding release files

Toolbox Migration/src/com/bayer/edam/excelimport/CropReleaseCopyTool.java

@@ -1,9 +1,11 @@
 package com.bayer.edam.excelimport;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.Layout;
@@ -24,10 +26,15 @@ import com.univocity.parsers.csv.CsvParserSettings;
  */
 public class CropReleaseCopyTool {
 	
+	private boolean dryRun = false;
+	
 	private static Logger log = Logger.getLogger(CropReleaseCopyTool.class);
 	private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.releasecopytool.log";
 	
-	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-23_incl_Releases.csv";
+	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv";
+	
+	private static String srcDir = "D:\\Toolbox\\ToolboxReleases\\";
+	private static String targetDir = "D:\\ToolboxReleases\\";
 	
 	private CsvParserSettings parserSettings = new CsvParserSettings();
 	private List<String[]> inputRows;
@@ -49,6 +56,20 @@ public class CropReleaseCopyTool {
 				if (!copiedReleases.contains(releaseId)) {
 					copiedReleases.add(releaseId);
 					log.info(line[0] + " '" + line[1] + "' --> " + line[28] + " '" + line[29] + "'");
+					
+					if (!dryRun) {
+						File srcFile = new File(srcDir + line[34]);
+						File targetFile = new File(targetDir + line[34]);
+						
+						try {
+							FileUtils.copyFile(srcFile, targetFile);
+						}
+						catch (IOException ioe) {
+							log.error("Line " + lineNo + ", AssetID " + line[0] + ", ReleaseID " + line[28] + " - IOException: '" + ioe.getMessage() + "'");
+							ioe.printStackTrace();
+						}
+					}
+
 				}
 			}
 		}

Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java

@@ -27,6 +27,7 @@ public class CropScienceAsset {
 	
 	HashSet<String> brands = new HashSet<String>();
 	HashSet<String> indications = new HashSet<String>();
+	HashSet<String> licenseRecords = new HashSet<String>();
 	
 	
 	Long fileSize;

Toolbox Migration/src/com/bayer/edam/excelimport/DirectoryListingCleaner.java

@@ -0,0 +1,59 @@
+package com.bayer.edam.excelimport;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.apache.log4j.RollingFileAppender;
+
+public class DirectoryListingCleaner {
+	
+	private static Logger log = Logger.getLogger(DirectoryListingCleaner.class);
+	private static String logfilePath = "D:\\ToolboxMigration-all.log";
+	
+	private static String listingFilePath = "D:\\ToolboxMigration-all.txt";
+	private List<String> filenames;
+	HashSet<String> uniqueFilenames = new HashSet<String>();
+	
+	public DirectoryListingCleaner() throws IOException {
+		filenames = FileUtils.readLines(new File(listingFilePath), "UTF-8");
+		
+		int lineNo = 0;
+		for (String filename : filenames) {
+			lineNo++;
+			if (!uniqueFilenames.contains(filename)) {
+				uniqueFilenames.add(filename);
+			}
+			else {
+				log.warn("Filename '" + filename + "' sorted out (line " + lineNo + ")");
+			}
+		}
+		
+		log.info("Unique filenames: " + uniqueFilenames.size());
+		
+	}
+
+
+	public static void main(String[] args) {
+		try {
+			Layout layout = new PatternLayout("%d{ISO8601} - %p %m%n"); // "%d{ISO8601} - %p %m%n"
+			BasicConfigurator.configure(new ConsoleAppender(layout));
+			BasicConfigurator.configure(new RollingFileAppender(layout, logfilePath, false));
+			log.setLevel(Level.INFO);
+
+			new DirectoryListingCleaner();
+
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+}

Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java

@@ -29,13 +29,14 @@ import com.univocity.parsers.csv.CsvWriterSettings;
 public class ToolboxImportFileGenerator {
 	private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class);
 
-	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.csv";
+	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.csv";
+	private static String licenseGuidsFilePath = "C:\\Temp\\Migration CS - SQL Server\\Release_GUIDs.csv";
 	
 	// XXX Spreadsheet needs be be named "Records" in Excel for the importer
-	private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files-IMPORT.csv";
+	private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases-IMPORT.csv";
 	
-	private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export_2017-08-22_missing_files.log";
+	private static String logfilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15-update_incl_Releases.log";
-	private static String uploadPath = "/Resources/Upload/ToolboxMigration2";
+	private static String uploadPath = "/Resources/Upload/ToolboxMigration";
 
 	private static final Map<String, String> brandMap;
 	private static final Map<String, String> unitMap;
@@ -974,6 +975,7 @@ public class ToolboxImportFileGenerator {
 	}
 
 	private List<String[]> inputRows;
+	private HashMap<String, String> licenses = new HashMap<String, String>();
 	private HashMap<Integer, CropScienceAsset> assets = new HashMap<Integer, CropScienceAsset>();
 	private CsvParserSettings parserSettings = new CsvParserSettings();
 	
@@ -981,7 +983,26 @@ public class ToolboxImportFileGenerator {
 	private SimpleDateFormat outputDateFormat = new SimpleDateFormat("MM/dd/yyyy");
 
 	public ToolboxImportFileGenerator() throws Exception {
-		readToolboxExport();
+		parserSettings.getFormat().setLineSeparator("\r\n");
+		parserSettings.getFormat().setDelimiter(';');
+		parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
+		parserSettings.setHeaderExtractionEnabled(false);
+
+		CsvParser parser = new CsvParser(parserSettings);
+
+		inputRows = parser.parseAll(new File(toolboxExportFilePath));
+		
+		List<String[]> licenseGuidRows = parser.parseAll(new File(licenseGuidsFilePath));
+		for (String[] line : licenseGuidRows) {
+			if (!licenses.containsKey(line[1])) {
+				licenses.put(line[1], line[0]);
+			}
+			else {
+				log.error("License " + line[1] + " has multiple keys");
+			}
+		}
+
+		
 		generateAssetDecriptions();
 		writeImportFile();
 	}
@@ -998,6 +1019,7 @@ public class ToolboxImportFileGenerator {
 		long internalAssets = 0;
 		long statusSkipped = 0;
 		long assetTypeNull = 0;
+		long releaseMissing = 0;
 
 		for (String[] line : inputRows) {
 			Integer assetId = Integer.parseInt(line[0]);
@@ -1152,9 +1174,18 @@ public class ToolboxImportFileGenerator {
 //						+ " of already assigned brand(s) " + assignedBrands);
 //			}
 			
-			// Indications stored in POJO for reference only. Indications are added to classifications in readToolboxExport()
 			if (indicationMap.get(line[14]) != null)
 				asset.indications.add(indicationMap.get(line[14]));
+			
+			if (!line[34].equals("NULL")) {
+				if (licenses.containsKey(line[34])) {
+					asset.licenseRecords.add(licenses.get(line[34]));
+				}
+				else {
+					log.error(asset.toolboxAssetId + ", line " + lineNo + ": No GUID for release file " + line[34]);
+					releaseMissing++;
+				}
+			}
 
 		}
 
@@ -1166,21 +1197,11 @@ public class ToolboxImportFileGenerator {
 		log.info("Lines Skipped: No target business unit: " + noTargetBusinessUnit);
 		log.info("Lines Skipped: Status is not 'Publish' or 'Archive': " + statusSkipped);
 		log.info("Lines Skipped: Asset type NULL: " + assetTypeNull);
+		log.info("Release missing: " + releaseMissing);
 		log.info("No asset type: " + noAssetType);
 		log.info("Internal assets: " + internalAssets);
 	}
 
-	private void readToolboxExport() {
-		parserSettings.getFormat().setLineSeparator("\r\n");
-		parserSettings.getFormat().setDelimiter(';');
-		parserSettings.getFormat().setCharToEscapeQuoteEscaping('"');
-		parserSettings.setHeaderExtractionEnabled(false);
-
-		CsvParser parser = new CsvParser(parserSettings);
-
-		inputRows = parser.parseAll(new File(toolboxExportFilePath));
-	}
-
 	private void writeImportFile() {
 		List<Object[]> csvOutputList = new ArrayList<Object[]>();
 		for (CropScienceAsset asset : assets.values()) {
@@ -1191,14 +1212,22 @@ public class ToolboxImportFileGenerator {
 					brands += "\n";
 				brands += brand;
 			}
+			
 			for (String indication : asset.indications)
 				asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication;
-
+			
+			String licenseRecords = "";
+			for (String licenseRecord : asset.licenseRecords) {
+				if (licenseRecords.length() > 0)
+					licenseRecords += "\n";
+				licenseRecords += licenseRecord;
+			}
+			
 			String[] csvLine = new String[] { asset.assetPath, asset.filename, asset.appendClassification, asset.byr_Title,
 					asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_businessUnit,
 					brands, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, asset.byr_cs_keywords_tl,
 					asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished,
-					asset.byr_cs_warehouseAvailable, asset.byr_cs_country };
+					asset.byr_cs_warehouseAvailable, asset.byr_cs_country, licenseRecords };
 			csvOutputList.add(csvLine);
 		}
 
@@ -1210,7 +1239,7 @@ public class ToolboxImportFileGenerator {
 				"fields|byr_cs_assetApproval", "fields|byr_cs_licenseType", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
 				"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
 				"fields|byr_campaignYearLabel", "fields|byr_campaignTitleLabel", "fields|byr_cs_IsShouldBePublished",
-				"fields|byr_cs_warehouseAvailable", "fields|byr_cs_country");
+				"fields|byr_cs_warehouseAvailable", "fields|byr_cs_country", "fields|byr_cs_licenseRecord");
 		writer.writeRowsAndClose(csvOutputList);
 	}