diff --git a/Toolbox Migration/src/com/bayer/edam/CropScienceAsset.java b/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java similarity index 83% rename from Toolbox Migration/src/com/bayer/edam/CropScienceAsset.java rename to Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java index 2ed8de2..a392a89 100644 --- a/Toolbox Migration/src/com/bayer/edam/CropScienceAsset.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/CropScienceAsset.java @@ -1,4 +1,4 @@ -package com.bayer.edam; +package com.bayer.edam.excelimport; import java.util.HashSet; @@ -13,7 +13,7 @@ public class CropScienceAsset { String byr_Title; String byr_MarketingId; String byr_cs_assetApproval; - String byr_cs_brand; + //String byr_cs_brand; String byr_cs_releaseDate; String byr_cs_retireDate; String byr_cs_businessUnit; @@ -25,6 +25,7 @@ public class CropScienceAsset { String byr_cs_IsShouldBePublished; String byr_cs_warehouseAvailable; + HashSet brands = new HashSet(); HashSet indications = new HashSet(); diff --git a/Toolbox Migration/src/com/bayer/edam/ToolboxImportFileGenerator.java b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java similarity index 90% rename from Toolbox Migration/src/com/bayer/edam/ToolboxImportFileGenerator.java rename to Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java index 7c7afd4..4bed47d 100644 --- a/Toolbox Migration/src/com/bayer/edam/ToolboxImportFileGenerator.java +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/ToolboxImportFileGenerator.java @@ -1,9 +1,10 @@ -package com.bayer.edam; +package com.bayer.edam.excelimport; import java.io.File; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -20,58 +21,14 @@ import com.univocity.parsers.csv.CsvWriter; import com.univocity.parsers.csv.CsvWriterSettings; /** - * Generates an import file from a toolbox CSV export. - * - *
- * SELECT
- * a.AssetId, a.OriginalFileName, a.UserFileName, a.BCSNumber, a.Description, a.AssetStatusID, a.PublishDate,
- * a.CreatedOn, a.CreatedByUserID, a.UpdatedOn, a.UpdatedByUserID, a.CopyClaimFileName
- * ,c.AssetCategoryName
- * ,ai.AssetItemName
- * ,f.FocusItemID
- * ,fi.FocusItemName
- * ,img.OriginalSize
- * ,a.publishDate
- * ,a.archiveDate
- * ,y.AssetCampaignYearName
- * ,p.ProjectName
- * ,a.isConfidential
- * ,u.FirstName
- * ,u.LastName
- * ,u.CompanyName
- * ,ff.FulfillmentCompanyName
- * ,ast.AssetStatusName      -- unused?
- * ,aty.AssetTypeName        -- unused?
- * 
- * FROM Assets a
- * LEFT JOIN AssetCategories c ON a.AssetCategoryID = c.AssetCategoryID
- * LEFT JOIN AssetItems ai ON a.AssetItemID = ai.AssetItemID
- * LEFT JOIN AssetFocusItemMap f ON a.AssetId = f.AssetID
- * LEFT JOIN FocusItems fi ON f.FocusItemID = fi.FocusItemID
- * LEFT JOIN AssetImages img ON a.AssetID = img.AssetID
- * LEFT JOIN AssetCampaignYears y ON a.AssetCampaignYearID = y.AssetCampaignYearID
- * LEFT JOIN Projects p ON a.ProjectID = p.ProjectID
- * LEFT JOIN Users u ON a.CreatedByUserID = u.UserID
- * LEFT JOIN FulfillmentCompanies ff ON a.FulfillmentCompanyID = ff.FulfillmentCompanyID
- * LEFT JOIN AssetStatuses ast ON a.AssetStatusID = ast.AssetStatusID
- * LEFT JOIN AssetTypes aty ON a.AssetTypeID = aty.AssetTypeID
- * 
- * WHERE 
- * a.AssetStatusID != 101
- * AND a.CreatedOn > Convert(datetime, '2012-07-01')
- * AND a.isDeleted = 0
- * AND (a.FocusID IS NULL OR a.FocusID != 3 OR (a.FocusID = 3 AND a.CreatedOn > Convert(datetime, '2016-01-01')))
- * --AND a.AssetID IN (23467,23392,22868,22488,22481,22440,22082,22021,21147,20731,
- * --19557,19048,18447,18446,18445,18444,16853,16171,15882,15313,14918,14400,
- * --13660,13641,13348,13323,13142,13135,12844,12814,12007)
- * 
+ * Generates an import file from a toolbox CSV export (see sql-code.sql) * * @author GEXCE */ public class ToolboxImportFileGenerator { private static Logger log = Logger.getLogger(ToolboxImportFileGenerator.class); - private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\auszug.csv"; + private static String toolboxExportFilePath = "C:\\Temp\\Migration CS - SQL Server\\export-2017-08-15.csv"; private static String importFilePath = "C:\\Temp\\Migration CS - SQL Server\\importFile.csv"; private static final Map brandMap; @@ -1024,38 +981,54 @@ public class ToolboxImportFileGenerator { private void generateAssetDecriptions() throws Exception { - long processedLines = 0; + long lineNo = 0; + HashSet processedAssetIDs = new HashSet(); long assetsIncluded = 0; long originalFilenameNull = 0; long noTargetBrand = 0; long noTargetBusinessUnit = 0; long noAssetType = 0; long internalAssets = 0; + long statusSkipped = 0; + long assetTypeNull = 0; for (String[] line : inputRows) { Integer assetId = Integer.parseInt(line[0]); log.debug("Processing AssetID " + assetId); - processedLines++; + lineNo++; + processedAssetIDs.add(assetId); // check exclusion criteria if (line[1].equals("NULL")) { - log.warn("Skipping AssetID " + assetId + " - OriginalFilename is 'NULL'"); + log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - OriginalFilename is 'NULL'"); originalFilenameNull++; continue; } - if (brandMap.get(line[15]) == null) { - log.warn("Skipping AssetID " + assetId + " - no target brand for '" + line[15] + "'"); + if (brandMap.get(line[14]) == null) { + log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - no target brand for '" + line[14] + "'"); noTargetBrand++; continue; } - if (unitMap.get(brandMap.get(line[15])) == null) { - log.warn("Skipping AssetID " + assetId + " - no target business unit for brand '" + brandMap.get(line[15]) + "'"); + if (unitMap.get(brandMap.get(line[14])) == null) { + log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - no target business unit for brand '" + brandMap.get(line[14]) + "'"); noTargetBusinessUnit++; continue; } + + if (!line[25].equals("Publish") && !line[25].equals("Archive")) { + log.info("Skipping AssetID " + assetId + ", line " + lineNo + " - status is '" + line[25] + "'"); + statusSkipped++; + continue; + } + + if (line[26].equals("NULL")) { + log.warn("Skipping AssetID " + assetId + ", line " + lineNo + " - asset type is " + line[26]); + assetTypeNull++; + continue; + } CropScienceAsset asset = assets.get(assetId); @@ -1068,36 +1041,24 @@ public class ToolboxImportFileGenerator { asset.byr_Title = line[2]; asset.byr_cs_assetApproval = "Approval Required"; - - if (!line[17].equals("NULL")) - asset.byr_cs_releaseDate = outputDateFormat.format(inputDateFormat.parse(line[17])); - if (!line[18].equals("NULL")) - asset.byr_cs_retireDate = outputDateFormat.format(inputDateFormat.parse(line[18])); - - asset.byr_cs_license_classification = "N/A"; - - if (line[25] != null && line[25].equals("HH Global")) - asset.byr_cs_warehouseAvailable = "Yes"; - else - asset.byr_cs_warehouseAvailable = "No"; - - asset.byr_cs_description = ""; - if (line[4] != null) - asset.byr_cs_description = line[4] + "\n\n"; - asset.byr_cs_description += "[Asset imported from Toolbox (ID " + asset.toolboxAssetId + "), created " - + outputDateFormat.format(inputDateFormat.parse(line[7])) + " by " + line[22] + " " + line[23] + " (" + line[24] - + ")]"; - - asset.byr_cs_keywords_tl = "toolbox import"; asset.byr_MarketingId = line[3]; - asset.byr_campaignYearLabel = line[19]; + asset.fileSize = Long.parseLong(line[15]); - if (!line[20].equals("NULL")) - asset.byr_campaignTitle = line[20]; + if (!line[16].equals("NULL")) + asset.byr_cs_releaseDate = outputDateFormat.format(inputDateFormat.parse(line[16])); + if (!line[17].equals("NULL")) + asset.byr_cs_retireDate = outputDateFormat.format(inputDateFormat.parse(line[17])); + + asset.byr_cs_license_classification = "N/A"; - if (line[21].equals("0")) { // "Internal" + asset.byr_campaignYearLabel = line[18]; + + if (!line[19].equals("NULL")) + asset.byr_campaignTitle = line[19]; + + if (line[20].equals("0")) { // "Internal" asset.byr_cs_IsShouldBePublished = "No"; } else { @@ -1105,43 +1066,81 @@ public class ToolboxImportFileGenerator { asset.byr_cs_IsShouldBePublished = "Yes"; } - asset.appendClassification = "/Definitions/Agencies/Bayer/Bayer CS\n" + "/Definitions/Common/Status/Released"; + if (line[24] != null && line[24].equals("HH Global")) + asset.byr_cs_warehouseAvailable = "Yes"; + else + asset.byr_cs_warehouseAvailable = "No"; + + + + /* description, keywords and classifications */ + + asset.byr_cs_description = ""; + if (line[4] != null) + asset.byr_cs_description = line[4] + "\n\n"; + asset.byr_cs_description += "[Asset imported from Toolbox (ID " + asset.toolboxAssetId + "), created " + + outputDateFormat.format(inputDateFormat.parse(line[6])) + " by " + line[21] + " " + line[22] + " (" + line[23] + + ")]"; + + asset.byr_cs_keywords_tl = "toolbox import"; + + asset.appendClassification = "/Definitions/Agencies/Bayer/Bayer CS"; + + if (line[25].equals("Publish")) + asset.appendClassification += "\n/Definitions/Common/Status/Released"; + else // "Archive" + asset.appendClassification += "\n/Definitions/Common/Status/Retired"; + + // determine asset type (from Toolbox "category") - String assetCategory = line[12] + " : " + line[13]; + String assetCategory = line[11] + " : " + line[12]; String targetType = categoryMap.get(assetCategory); if (targetType != null) { asset.appendClassification = asset.appendClassification - + "\n/Bayer/Crop Science/Categroy\\/Item/" + targetType.substring(0, targetType.indexOf(" : ")) - + "\n/Bayer/Crop Science/Categroy\\/Item/" + targetType.substring(targetType.indexOf(" : ") + 3); + + "\n/Bayer/Crop Science/Category\\/Item/" + targetType.substring(0, targetType.indexOf(" : ")) + + "\n/Bayer/Crop Science/Category\\/Item/" + targetType.substring(targetType.indexOf(" : ") + 3); } else { noAssetType++; - log.info(asset.toolboxAssetId + ": Could not map category '" + assetCategory + "' to any asset type"); + log.warn(asset.toolboxAssetId + ", line " + lineNo + ": Could not map category '" + assetCategory + "' to any asset type"); } - asset.fileSize = Long.parseLong(line[16]); - assets.put(assetId, asset); } - if (asset.byr_cs_brand == null) { - asset.byr_cs_brand = brandMap.get(line[15]); - asset.byr_cs_businessUnit = unitMap.get(asset.byr_cs_brand); - } else { - asset.byr_cs_brand = asset.byr_cs_brand + "\n" + line[15]; + asset.brands.add(brandMap.get(line[14])); + if (asset.byr_cs_businessUnit == null) { + asset.byr_cs_businessUnit = unitMap.get(brandMap.get(line[14])); + } + else if (!asset.byr_cs_businessUnit.equals(unitMap.get(brandMap.get(line[14])))) { + + String assignedBrands = ""; + for (String brand : asset.brands) { + if (assignedBrands.length() > 0) + assignedBrands += ", "; + assignedBrands += brand; + } + + log.error("Asset " + asset.toolboxAssetId + ": Additional brand " + brandMap.get(line[14]) + " (" + + unitMap.get(brandMap.get(line[14])) + ") does not belong to business unit " + asset.byr_cs_businessUnit + + " of already assigned brand(s) " + assignedBrands); } - if (indicationMap.get(line[15]) != null) - asset.indications.add(indicationMap.get(line[15])); + // Indications stored in POJO for reference only. Indications are added to classifications in readToolboxExport() + if (indicationMap.get(line[14]) != null) + asset.indications.add(indicationMap.get(line[14])); } - log.info("Processed lines: " + processedLines); - log.info("Assets included: " + assetsIncluded); - log.info("Skipped: Original filname NULL: " + originalFilenameNull); - log.info("Skipped: No target brand: " + noTargetBrand); - log.info("Skipped: No target business unit: " + noTargetBusinessUnit); + log.info("Processed lines : " + lineNo); + log.info("Processed assets : " + processedAssetIDs.size()); + log.info("Assets included : " + assetsIncluded); + log.info("Lines Skipped: Original filname NULL: " + originalFilenameNull); + log.info("Lines Skipped: No target brand: " + noTargetBrand); + log.info("Lines Skipped: No target business unit: " + noTargetBusinessUnit); + log.info("Lines Skipped: Status is not 'Publish' or 'Archive': " + statusSkipped); + log.info("Lines Skipped: Asset type NULL: " + assetTypeNull); log.info("No asset type: " + noAssetType); log.info("Internal assets: " + internalAssets); } @@ -1160,10 +1159,18 @@ public class ToolboxImportFileGenerator { private void writeImportFile() { List csvOutputList = new ArrayList(); for (CropScienceAsset asset : assets.values()) { + + String brands = ""; + for (String brand : asset.brands) + brands += brand; + for (String indication : asset.indications) + asset.appendClassification += "\n/Bayer/Crop Science/Indication/" + indication; + String[] csvLine = new String[] { asset.assetPath, asset.filename, asset.appendClassification, asset.byr_Title, - asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_brand, - asset.byr_cs_businessUnit, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, - asset.byr_cs_keywords_tl, asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished, asset.byr_cs_warehouseAvailable }; + asset.byr_MarketingId, asset.byr_cs_assetApproval, asset.byr_cs_license_classification, asset.byr_cs_businessUnit, + brands, asset.byr_cs_releaseDate, asset.byr_cs_retireDate, asset.byr_cs_description, asset.byr_cs_keywords_tl, + asset.byr_campaignYearLabel, asset.byr_campaignTitle, asset.byr_cs_IsShouldBePublished, + asset.byr_cs_warehouseAvailable }; csvOutputList.add(csvLine); } @@ -1172,7 +1179,7 @@ public class ToolboxImportFileGenerator { CsvWriter writer = new CsvWriter(new File(importFilePath), writerSettings); writer.writeHeaders("assetpath", "filename", "append classification", "fields|byr_title", "fields|byr_MarketingId", - "fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_brand", "fields|byr_cs_businessUnit", + "fields|byr_cs_assetApproval", "fields|byr_cs_license_classification", "fields|byr_cs_businessUnit", "fields|byr_cs_brand", "fields|byr_cs_releaseDate", "fields|byr_cs_retireDate", "fields|byr_cs_Description", "fields|byr_cs_keywords_tl", "fields|byr_campaignYearLabel", "fields|byr_campaignTitle", "fields|byr_cs_IsShouldBePublished", "fields|byr_cs_warehouseAvailable"); diff --git a/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql b/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql new file mode 100644 index 0000000..de448c7 --- /dev/null +++ b/Toolbox Migration/src/com/bayer/edam/excelimport/sql-code.sql @@ -0,0 +1,57 @@ +SELECT DISTINCT +a.AssetId, a.OriginalFileName, a.UserFileName, a.BCSNumber, a.Description, a.AssetStatusID, +a.CreatedOn, a.CreatedByUserID, a.UpdatedOn, a.UpdatedByUserID, a.CopyClaimFileName +,c.AssetCategoryName +,ai.AssetItemName +,f.FocusItemID +,fi.FocusItemName +,img.OriginalSize +,a.publishDate +,a.archiveDate +,y.AssetCampaignYearName +,p.ProjectName +,a.isConfidential +,u.FirstName +,u.LastName +,u.CompanyName +,ff.FulfillmentCompanyName +,ast.AssetStatusName +,aty.AssetTypeName +,org.OrganizationName + +FROM Assets a +LEFT JOIN AssetCategories c ON a.AssetCategoryID = c.AssetCategoryID +LEFT JOIN AssetItems ai ON a.AssetItemID = ai.AssetItemID +LEFT JOIN AssetFocusItemMap f ON a.AssetId = f.AssetID +LEFT JOIN FocusItems fi ON f.FocusItemID = fi.FocusItemID +LEFT JOIN AssetImages img ON a.AssetID = img.AssetID +LEFT JOIN AssetCampaignYears y ON a.AssetCampaignYearID = y.AssetCampaignYearID +LEFT JOIN Projects p ON a.ProjectID = p.ProjectID +LEFT JOIN Users u ON a.CreatedByUserID = u.UserID +LEFT JOIN FulfillmentCompanies ff ON a.FulfillmentCompanyID = ff.FulfillmentCompanyID +LEFT JOIN AssetStatuses ast ON a.AssetStatusID = ast.AssetStatusID +LEFT JOIN AssetTypes aty ON a.AssetTypeID = aty.AssetTypeID + + +-- all these joins only to get the "Available to" matrix +LEFT JOIN AssetGroupSecurityFunctionMap agsfm ON a.AssetID = agsfm.AssetID +LEFT JOIN Organizations org ON org.OrganizationID = agsfm.OrganizationID +LEFT JOIN Groups grp ON grp.GroupID = agsfm.GroupID +--LEFT JOIN SecurityFunctions sf ON sf.SecurityFunctionID = agsfm.SecurityFunctionID + + +WHERE +a.AssetStatusID != 101 +AND a.CreatedOn > Convert(datetime, '2012-07-01') +AND a.isDeleted = 0 +AND (a.FocusID IS NULL OR a.FocusID != 3 OR (a.FocusID = 3 AND a.CreatedOn > Convert(datetime, '2016-01-01'))) +AND grp.GroupName != 'Administrator' + +/* +AND a.AssetID IN (24024,24014,24007,23933,23932,23830,23826,23805,23804, +23803,23802,23801,23800,23799,23797,23751,23707,23688,23657,23629,23622, +23517,23484,23483,23472,23470,23467,23411,23392,23381,23211,22868,22488, +22481,22440,22082,22021,21147,20731,19557,19048,18447,18446,18445,18444, +16853,16171,15882,15313,14918,14400,13660,13641,13348,13323,13142,13135, +12844,12814,12007) +*/