everything but licenses

Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java

@@ -1,4 +1,4 @@
-package com.bayer.edam;
+package com.bayer.edam.excelimport;
 
 import java.util.HashSet;
 
@@ -13,7 +13,7 @@ public class CropScienceAsset {
 	String byr_Title;
 	String byr_MarketingId;
 	String byr_cs_assetApproval;
-	String byr_cs_brand;
+	//String byr_cs_brand;
 	String byr_cs_releaseDate;
 	String byr_cs_retireDate;
 	String byr_cs_businessUnit;
@@ -25,6 +25,7 @@ public class CropScienceAsset {
 	String byr_cs_IsShouldBePublished;
 	String byr_cs_warehouseAvailable;
 	
+	HashSet<String> brands = new HashSet<String>();
 	HashSet<String> indications = new HashSet<String>();
 	
 	

Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java

@@ -1,9 +1,10 @@
-package com.bayer.edam;
+package com.bayer.edam.excelimport;
 
 import java.io.File;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -20,58 +21,14 @@ import com.univocity.parsers.csv.CsvWriter;
 import com.univocity.parsers.csv.CsvWriterSettings;
 
 /**
- * Generates an import file from a toolbox CSV export.
+ * Generates an import file from a toolbox CSV export (see <a href="sql-code.sql">sql-code.sql</a>)
- *
- * <pre>
- * SELECT
- * a.AssetId, a.OriginalFileName, a.UserFileName, a.BCSNumber, a.Description, a.AssetStatusID, a.PublishDate,
- * a.CreatedOn, a.CreatedByUserID, a.UpdatedOn, a.UpdatedByUserID, a.CopyClaimFileName
- * ,c.AssetCategoryName
- * ,ai.AssetItemName
- * ,f.FocusItemID
- * ,fi.FocusItemName
- * ,img.OriginalSize
- * ,a.publishDate
- * ,a.archiveDate
- * ,y.AssetCampaignYearName
- * ,p.ProjectName
- * ,a.isConfidential
- * ,u.FirstName
- * ,u.LastName
- * ,u.CompanyName
- * ,ff.FulfillmentCompanyName
- * ,ast.AssetStatusName      -- unused?
- * ,aty.AssetTypeName        -- unused?
- * 
- * FROM Assets a
- * LEFT JOIN AssetCategories c ON a.AssetCategoryID = c.AssetCategoryID
- * LEFT JOIN AssetItems ai ON a.AssetItemID = ai.AssetItemID
- * LEFT JOIN AssetFocusItemMap f ON a.AssetId = f.AssetID
- * LEFT JOIN FocusItems fi ON f.FocusItemID = fi.FocusItemID
- * LEFT JOIN AssetImages img ON a.AssetID = img.AssetID
- * LEFT JOIN AssetCampaignYears y ON a.AssetCampaignYearID = y.AssetCampaignYearID
- * LEFT JOIN Projects p ON a.ProjectID = p.ProjectID
- * LEFT JOIN Users u ON a.CreatedByUserID = u.UserID
- * LEFT JOIN FulfillmentCompanies ff ON a.FulfillmentCompanyID = ff.FulfillmentCompanyID
- * LEFT JOIN AssetStatuses ast ON a.AssetStatusID = ast.AssetStatusID
- * LEFT JOIN AssetTypes aty ON a.AssetTypeID = aty.AssetTypeID
- * 
- * WHERE 
- * a.AssetStatusID != 101
- * AND a.CreatedOn > Convert(datetime, '2012-07-01')
- * AND a.isDeleted = 0
- * AND (a.FocusID IS NULL OR a.FocusID != 3 OR (a.FocusID = 3 AND a.CreatedOn > Convert(datetime, '2016-01-01')))
- * --AND a.AssetID IN (23467,23392,22868,22488,22481,22440,22082,22021,21147,20731,
- * --19557,19048,18447,18446,18445,18444,16853,16171,15882,15313,14918,14400,
- * --13660,13641,13348,13323,13142,13135,12844,12814,12007)
- * </pre>
  * 
  * @author GEXCE
  */
 public class ToolboxImportFileGenerator {
 	private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class);
 
-	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\auszug.csv";
+	private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv";
 	private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile.csv";
 
 	private static final Map<String, String> brandMap;
@@ -1024,39 +981,55 @@ public class ToolboxImportFileGenerator {
 
 	private void generateAssetDecriptions() throws Exception {
 
-		long processedLines = 0;
+		long lineNo = 0;
+		HashSet<Integer> processedAssetIDs = new HashSet<Integer>();
 		long assetsIncluded = 0;
 		long originalFilenameNull = 0;
 		long noTargetBrand = 0;
 		long noTargetBusinessUnit = 0;
 		long noAssetType = 0;
 		long internalAssets = 0;
+		long statusSkipped = 0;
+		long assetTypeNull = 0;
 
 		for (String[] line : inputRows) {
 			Integer assetId = Integer.parseInt(line[0]);
 
 			log.debug("Processing AssetID " + assetId);
-			processedLines++;
+			lineNo++;
+			processedAssetIDs.add(assetId);
 
 			// check exclusion criteria
 			if (line[1].equals("NULL")) {
-				log.warn("Skipping AssetID " + assetId + " - OriginalFilename is 'NULL'");
+				log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - OriginalFilename is 'NULL'");
 				originalFilenameNull++;
 				continue;
 			}
 
-			if (brandMap.get(line[15]) == null) {
+			if (brandMap.get(line[14]) == null) {
-				log.warn("Skipping AssetID " + assetId + " - no target brand for '" + line[15] + "'");
+				log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - no target brand for '" + line[14] + "'");
 				noTargetBrand++;
 				continue;
 			}
 
-			if (unitMap.get(brandMap.get(line[15])) == null) {
+			if (unitMap.get(brandMap.get(line[14])) == null) {
-				log.warn("Skipping AssetID " + assetId + " - no target business unit for brand '" + brandMap.get(line[15]) + "'");
+				log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - no target business unit for brand '" + brandMap.get(line[14]) + "'");
 				noTargetBusinessUnit++;
 				continue;
 			}
 			
+			if (!line[25].equals("Publish") && !line[25].equals("Archive")) {
+				log.info("Skipping AssetID " + assetId + ", line " + lineNo + " - status is '" + line[25] + "'");
+				statusSkipped++;
+				continue;
+			}
+
+			if (line[26].equals("NULL")) {
+				log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - asset type is " + line[26]);
+				assetTypeNull++;
+				continue;
+			}
+
 			CropScienceAsset asset = assets.get(assetId);
 
 			if (asset == null) { // Fields|byr_cs_country
@@ -1069,35 +1042,23 @@ public class ToolboxImportFileGenerator {
 				asset.byr_Title = line[2];
 				asset.byr_cs_assetApproval = "Approval Required";
 
+				asset.byr_MarketingId = line[3];
+				
+				asset.fileSize = Long.parseLong(line[15]);
+				
+				if (!line[16].equals("NULL"))
+					asset.byr_cs_releaseDate = outputDateFormat.format(inputDateFormat.parse(line[16]));
 				if (!line[17].equals("NULL"))
-					asset.byr_cs_releaseDate = outputDateFormat.format(inputDateFormat.parse(line[17]));
+					asset.byr_cs_retireDate = outputDateFormat.format(inputDateFormat.parse(line[17]));
-				if (!line[18].equals("NULL"))
-					asset.byr_cs_retireDate = outputDateFormat.format(inputDateFormat.parse(line[18]));
 
 				asset.byr_cs_license_classification = "N/A";
 				
-				if (line[25] != null && line[25].equals("HH Global"))
+				asset.byr_campaignYearLabel = line[18];
-					asset.byr_cs_warehouseAvailable = "Yes";
-				else
-					asset.byr_cs_warehouseAvailable = "No";
 				
-				asset.byr_cs_description = "";
+				if (!line[19].equals("NULL"))
-				if (line[4] != null)
+					asset.byr_campaignTitle = line[19];
-					asset.byr_cs_description = line[4] + "\n\n";
-				asset.byr_cs_description += "[Asset imported from Toolbox (ID " + asset.toolboxAssetId + "), created "
-						+ outputDateFormat.format(inputDateFormat.parse(line[7])) + " by " + line[22] + " " + line[23] + " (" + line[24]
-						+ ")]";
 				
-				asset.byr_cs_keywords_tl = "toolbox import";
+				if (line[20].equals("0")) { // "Internal"
-
-				asset.byr_MarketingId = line[3];
-				
-				asset.byr_campaignYearLabel = line[19];
-				
-				if (!line[20].equals("NULL"))
-					asset.byr_campaignTitle = line[20];
-				
-				if (line[21].equals("0")) { // "Internal"
 					asset.byr_cs_IsShouldBePublished = "No";
 				}
 				else {
@@ -1105,43 +1066,81 @@ public class ToolboxImportFileGenerator {
 					asset.byr_cs_IsShouldBePublished = "Yes";
 				}
 				
-				asset.appendClassification = "/Definitions/Agencies/Bayer/Bayer CS\n" + "/Definitions/Common/Status/Released";
+				if (line[24] != null && line[24].equals("HH Global"))
+					asset.byr_cs_warehouseAvailable = "Yes";
+				else
+					asset.byr_cs_warehouseAvailable = "No";
+				
+				
+				
+				/* description, keywords and classifications */
+				
+				asset.byr_cs_description = "";
+				if (line[4] != null)
+					asset.byr_cs_description = line[4] + "\n\n";
+				asset.byr_cs_description += "[Asset imported from Toolbox (ID " + asset.toolboxAssetId + "), created "
+						+ outputDateFormat.format(inputDateFormat.parse(line[6])) + " by " + line[21] + " " + line[22] + " (" + line[23]
+						+ ")]";
+				
+				asset.byr_cs_keywords_tl = "toolbox import";
+				
+				asset.appendClassification = "/Definitions/Agencies/Bayer/Bayer CS";
+				
+				if (line[25].equals("Publish"))
+					asset.appendClassification += "\n/Definitions/Common/Status/Released";
+				else // "Archive"
+					asset.appendClassification += "\n/Definitions/Common/Status/Retired";
+					
+					
 				
 				// determine asset type (from Toolbox "category")
-				String assetCategory = line[12] + " : " + line[13];
+				String assetCategory = line[11] + " : " + line[12];
 				String targetType = categoryMap.get(assetCategory);
 				if (targetType != null) {
 					asset.appendClassification = asset.appendClassification
-							+ "\n/Bayer/Crop Science/Categroy\\/Item/" + targetType.substring(0, targetType.indexOf(" : "))
+							+ "\n/Bayer/Crop Science/Category\\/Item/" + targetType.substring(0, targetType.indexOf(" : "))
-							+ "\n/Bayer/Crop Science/Categroy\\/Item/" + targetType.substring(targetType.indexOf(" : ") + 3);
+							+ "\n/Bayer/Crop Science/Category\\/Item/" + targetType.substring(targetType.indexOf(" : ") + 3);
 				}
 				else {
 					noAssetType++;
-					log.info(asset.toolboxAssetId + ": Could not map category '" + assetCategory + "' to any asset type");
+					log.warn(asset.toolboxAssetId + ", line " + lineNo + ": Could not map category '" + assetCategory + "' to any asset type");
 				}
 
-				asset.fileSize = Long.parseLong(line[16]);
-
 				assets.put(assetId, asset);
 			}
 
-			if (asset.byr_cs_brand == null) {
+			asset.brands.add(brandMap.get(line[14]));
-				asset.byr_cs_brand = brandMap.get(line[15]);
+			if (asset.byr_cs_businessUnit == null) {
-				asset.byr_cs_businessUnit = unitMap.get(asset.byr_cs_brand);
+				asset.byr_cs_businessUnit = unitMap.get(brandMap.get(line[14]));
-			} else {
+			}
-				asset.byr_cs_brand = asset.byr_cs_brand + "\n" + line[15];
+			else if (!asset.byr_cs_businessUnit.equals(unitMap.get(brandMap.get(line[14])))) {
+				
+				String assignedBrands = "";
+				for (String brand : asset.brands) {
+					if (assignedBrands.length() > 0)
+						assignedBrands += ", ";
+					assignedBrands += brand;
 				}
 				
-			if (indicationMap.get(line[15]) != null)
+				log.error("Asset " + asset.toolboxAssetId + ": Additional brand " + brandMap.get(line[14]) + " ("
-				asset.indications.add(indicationMap.get(line[15]));
+						+ unitMap.get(brandMap.get(line[14])) + ") does not belong to business unit " + asset.byr_cs_businessUnit
+						+ " of already assigned brand(s) " + assignedBrands);
+			}
+			
+			// Indications stored in POJO for reference only. Indications are added to classifications in readToolboxExport()
+			if (indicationMap.get(line[14]) != null)
+				asset.indications.add(indicationMap.get(line[14]));
 
 		}
 
-		log.info("Processed lines: " + processedLines);
+		log.info("Processed lines   : " + lineNo);
+		log.info("Processed assets  : " + processedAssetIDs.size());
 		log.info("Assets included   : " + assetsIncluded);
-		log.info("Skipped: Original filname NULL: " + originalFilenameNull);
+		log.info("Lines Skipped: Original filname NULL: " + originalFilenameNull);
-		log.info("Skipped: No target brand: " + noTargetBrand);
+		log.info("Lines Skipped: No target brand: " + noTargetBrand);
-		log.info("Skipped: No target business unit: " + noTargetBusinessUnit);
+		log.info("Lines Skipped: No target business unit: " + noTargetBusinessUnit);
+		log.info("Lines Skipped: Status is not 'Publish' or 'Archive': " + statusSkipped);
+		log.info("Lines Skipped: Asset type NULL: " + assetTypeNull);
 		log.info("No asset type: " + noAssetType);
 		log.info("Internal assets: " + internalAssets);
 	}
@@ -1160,10 +1159,18 @@ public class ToolboxImportFileGenerator {
 	private void writeImportFile() {
 		List<Object[]> csvOutputList = new ArrayList<Object[]>();
 		for (CropScienceAsset asset : assets.values()) {
+
+			String brands = "";
+			for (String brand : asset.brands)
+				brands += brand;
+			for (String indication : asset.indications)
+				asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication;
+
 			String[] csvLine = new String[] { asset.assetPath, asset.filename, asset.appendClassification, asset.byr_Title,
-					asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_brand,
+					asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_businessUnit,
-					asset.byr_cs_businessUnit, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description,
+					brands, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, asset.byr_cs_keywords_tl,
-					asset.byr_cs_keywords_tl, asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished, asset.byr_cs_warehouseAvailable };
+					asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished,
+					asset.byr_cs_warehouseAvailable };
 			csvOutputList.add(csvLine);
 		}
 
@@ -1172,7 +1179,7 @@ public class ToolboxImportFileGenerator {
 
 		CsvWriter writer = new CsvWriter(new File(importFilePath), writerSettings);
 		writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId",
-				"fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_brand", "fields|byr_cs_businessUnit",
+				"fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand",
 				"fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl",
 				"fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished",
 				"fields|byr_cs_warehouseAvailable");

Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql

@@ -0,0 +1,57 @@
+SELECT DISTINCT
+a.AssetId, a.OriginalFileName, a.UserFileName, a.BCSNumber, a.Description, a.AssetStatusID, 
+a.CreatedOn, a.CreatedByUserID, a.UpdatedOn, a.UpdatedByUserID, a.CopyClaimFileName
+,c.AssetCategoryName
+,ai.AssetItemName
+,f.FocusItemID
+,fi.FocusItemName
+,img.OriginalSize
+,a.publishDate
+,a.archiveDate
+,y.AssetCampaignYearName
+,p.ProjectName
+,a.isConfidential
+,u.FirstName
+,u.LastName
+,u.CompanyName
+,ff.FulfillmentCompanyName
+,ast.AssetStatusName
+,aty.AssetTypeName
+,org.OrganizationName
+
+FROM Assets a
+LEFT JOIN AssetCategories c ON a.AssetCategoryID = c.AssetCategoryID
+LEFT JOIN AssetItems ai ON a.AssetItemID = ai.AssetItemID
+LEFT JOIN AssetFocusItemMap f ON a.AssetId = f.AssetID
+LEFT JOIN FocusItems fi ON f.FocusItemID = fi.FocusItemID
+LEFT JOIN AssetImages img ON a.AssetID = img.AssetID
+LEFT JOIN AssetCampaignYears y ON a.AssetCampaignYearID = y.AssetCampaignYearID
+LEFT JOIN Projects p ON a.ProjectID = p.ProjectID
+LEFT JOIN Users u ON a.CreatedByUserID = u.UserID
+LEFT JOIN FulfillmentCompanies ff ON a.FulfillmentCompanyID = ff.FulfillmentCompanyID
+LEFT JOIN AssetStatuses ast ON a.AssetStatusID = ast.AssetStatusID
+LEFT JOIN AssetTypes aty ON a.AssetTypeID = aty.AssetTypeID
+
+
+-- all these joins only to get the "Available to" matrix
+LEFT JOIN AssetGroupSecurityFunctionMap agsfm ON a.AssetID = agsfm.AssetID
+LEFT JOIN Organizations org ON org.OrganizationID = agsfm.OrganizationID
+LEFT JOIN Groups grp ON grp.GroupID = agsfm.GroupID
+--LEFT JOIN SecurityFunctions sf ON sf.SecurityFunctionID = agsfm.SecurityFunctionID
+
+
+WHERE 
+a.AssetStatusID != 101
+AND a.CreatedOn > Convert(datetime, '2012-07-01')
+AND a.isDeleted = 0
+AND (a.FocusID IS NULL OR a.FocusID != 3 OR (a.FocusID = 3 AND a.CreatedOn > Convert(datetime, '2016-01-01')))
+AND grp.GroupName != 'Administrator'
+
+/*
+AND a.AssetID IN (24024,24014,24007,23933,23932,23830,23826,23805,23804,
+23803,23802,23801,23800,23799,23797,23751,23707,23688,23657,23629,23622,
+23517,23484,23483,23472,23470,23467,23411,23392,23381,23211,22868,22488,
+22481,22440,22082,22021,21147,20731,19557,19048,18447,18446,18445,18444,
+16853,16171,15882,15313,14918,14400,13660,13641,13348,13323,13142,13135,
+12844,12814,12007)
+*/