From 5944134afcb521af778028e6e342afff7168eed3 Mon Sep 17 00:00:00 2001 From: tliero Date: Wed, 5 Jul 2017 09:04:46 +0200 Subject: [PATCH] incl. File.exists() check --- .../edam/ToolboxDirectVideoExporter.java | 190 +++++++++++++----- 1 file changed, 141 insertions(+), 49 deletions(-) diff --git a/Toolbox Migration/src/com/bayer/edam/ToolboxDirectVideoExporter.java b/Toolbox Migration/src/com/bayer/edam/ToolboxDirectVideoExporter.java index cd171ce..45c4427 100644 --- a/Toolbox Migration/src/com/bayer/edam/ToolboxDirectVideoExporter.java +++ b/Toolbox Migration/src/com/bayer/edam/ToolboxDirectVideoExporter.java @@ -1,6 +1,8 @@ package com.bayer.edam; -import java.io.UnsupportedEncodingException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -10,25 +12,86 @@ import org.apache.http.NameValuePair; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.RollingFileAppender; public class ToolboxDirectVideoExporter { + private static Logger log = Logger.getLogger(ToolboxDirectVideoExporter.class); + String cookie = "ASP.NET_SessionId=kboqi1uznhyvxsgyezsl5z3t; userid=1041"; boolean proxy = true; + static final String viewStateMarker = "id=\"__VIEWSTATE\" value=\""; + static final String eventValidationMarker = "id=\"__EVENTVALIDATION\" value=\""; + static final String baseDir = "F:\\Toolbox\\Movies\\"; + CloseableHttpClient httpClient = HttpClients.createDefault(); HttpHost target = new HttpHost("bayertoolbox.com", 443, "https"); HttpPost httpPost; + HttpGet httpGet; HttpEntity httpEntity; RequestConfig config; + CloseableHttpResponse response; + List urlParameters; + +// int[] testId = {20553, 3439, 23445, 11428}; + + int[] assetId = {19664, 20553, 4060, 10042, 9659, 9660, 9658, 10041, 9652, 9656, 4135, 19332, 19488, 10483, 11924, 1392, + 11923, 10482, 8071, 1396, 10481, 11922, 3200, 11920, 10479, 20711, 10472, 11913, 10607, 3342, 9657, 3201, 11921, + 10480, 1382, 4046, 10478, 11919, 11350, 1393, 10355, 10474, 11915, 1395, 10490, 11788, 11789, 10491, 3204, 3202, 3338, + 15700, 11917, 10476, 10475, 11916, 10745, 10752, 10785, 13996, 11332, 13723, 11912, 10471, 20961, 13722, 11428, 15397, + 11353, 2396, 10762, 2398, 11787, 10489, 3431, 10473, 11914, 10606, 13459, 13719, 11772, 3184, 4264, 13758, 2234, 1435, + 11894, 11897, 3205, 11352, 9655, 13997, 3203, 20502, 11899, 13730, 10500, 11801, 2188, 11327, 1442, 2387, 4124, 4115, + 4239, 3198, 3199, 11336, 10551, 11335, 10292, 10550, 10553, 10291, 9653, 9654, 10295, 10545, 10286, 11807, 11804, + 11344, 11774, 20506, 11328, 2399, 1439, 11348, 13994, 14136, 3442, 11349, 11902, 11270, 22936, 11896, 1394, 20503, + 11949, 2395, 10495, 11793, 11900, 3434, 1039, 1467, 10608, 20504, 11785, 11786, 10488, 11287, 23289, 11895, 11893, + 11773, 20501, 19579, 23510, 1441, 22937, 23288, 23490, 14322, 13726, 20507, 11797, 11798, 10496, 13721, 20505, 3432, + 19382, 20200, 10356, 11951, 19491, 23233, 11117, 19498, 11790, 10492, 23643, 11824, 11898, 11802, 10501, 19493, 18843, + 3441, 4263, 23086, 10746, 4535, 11901, 11952, 19492, 19487, 22947, 23870, 23783, 19500, 11820, 23550, 19490, 10384, + 19495, 19489, 19494, 19496, 11918, 10477, 19486, 11347, 3339, 19499, 11285, 11346, 13995, 11821, 19917, 1443, 11950, + 13589, 3206, 23997, 11780, 11781, 10485, 11822, 14160, 13728, 11910, 23756, 20059, 23697, 11838, 14162, 14151, 13720, + 11827, 13991, 11962, 11823, 23839, 23209, 11830, 1438, 11826, 11967, 10605, 11825, 11828, 10487, 11783, 11784, 11337, + 11891, 23087, 11928, 23236, 11333, 11887, 11808, 11791, 10493, 11836, 11905, 11961, 10499, 11800, 11875, 11884, 13115, + 11528, 11829, 11832, 10761, 2386, 11286, 1440, 10294, 11926, 12950, 10554, 11946, 10296, 3440, 10549, 10290, 12948, + 11964, 11176, 10548, 10289, 21024, 11429, 10544, 11839, 10494, 11792, 10285, 14474, 21013, 11878, 11831, 3435, 11873, + 14265, 2393, 11909, 20731, 11782, 10486, 13731, 11338, 11835, 11879, 11965, 11943, 19329, 2397, 11345, 14161, 11968, + 2392, 11956, 11953, 3337, 11889, 13458, 11813, 11882, 11906, 11954, 11963, 11958, 11834, 3433, 11970, 14321, 13467, + 13468, 11343, 11840, 11837, 11957, 10497, 11890, 11871, 1380, 11799, 10498, 11118, 2480, 2494, 13725, 11810, 11811, + 11812, 13729, 10357, 11904, 3439, 11969, 13471, 11945, 11877, 11929, 14589, 11886, 11888, 11960, 11814, 11966, 11326, + 10760, 21266, 10361, 11941, 11883, 11959, 20545, 11324, 11325, 11179, 20546, 20421, 1437, 11334, 20422, 11927, 11876, + 20484, 11341, 23617, 11880, 23445, 23446, 3437, 11870, 11938, 11364, 11872, 11944, 11815, 11925, 11833, 11881, 11885, + 11809, 19538, 19919, 11331, 19934, 18793, 3438, 11935, 11351, 11942, 12141, 11329, 11818, 4134, 11936, 11330, 11930, + 11340, 3347, 12162, 11908, 11937, 11816, 11940, 11932, 11819, 11874, 11869, 11971, 19060, 11933, 11750, 13590, 12403, + 12215, 19497, 11939, 22148, 3368, 23683, 13933, 14166, 3436, 14152, 14163, 11342, 11817, 23716, 10383, 11907, 23615, + 13470, 23714, 11972, 19333, 20668, 23682, 13469, 11903, 21396, 23613, 23612, 23718, 11934, 23836, 23497, 2391, 23713, + 11973, 2078, 14159, 13727, 23499, 23829, 23500, 10205, 23616, 2394, 10108, 14900, 2385, 20938, 14409, 23181, 23439, + 23831, 23840, 21006, 3430, 23832, 23837, 14191, 14907, 23868, 23370, 23614, 11931, 23833, 23830, 23841, 3134, 19704, + 19918, 23838, 13992, 23834, 13513, 19576, 13113, 13993, 13512, 2382, 19922, 19920, 19921, 2383, 19650, 11339, 23681, + 18724, 14411, 8893, 23712, 15288, 23314, 23024, 14412, 21014, 19963, 23717, 19480, 23715, 23372, 8815, 14410, 19832, + 18569, 18722, 19787, 10987, 10988, 11050, 2600, 21335, 21347, 21302, 12949, 18721, 18568, 13426, 13694, 13724, 11432, + 11361, 11436, 11435, 11173, 11433, 11434, 11431, 11178, 13511, 11175, 13509, 20960, 21303, 13427, 13425, 20937, 10034, + 15320, 22171, 20972, 10359, 13424, 18723, 20555, 13423, 11097, 13361, 11753, 13305, 19482, 19476, 23279, 23280, 10353, + 23089, 24418, 2079, 10354, 23281, 10360, 23276, 10148, 24417, 23048, 24419, 12868, 10149, 13306, 13275, 10358, 13301, + 21235, 12805, 13304, 22959, 11048, 1041, 20635, 20636, 23546, 24420, 12585, 19277, 3984, 3491, 24274, 2401, 22525, + 13323, 8803, 23648, 10546, 10287, 21900, 23680, 2402, 23633, 22646, 21899, 24276, 11446, 23795, 985, 2390, 24275, + 22464, 23610, 21883, 1040, 10609, 23474, 23746, 19475, 19484, 22046, 19472, 19481, 22047, 19478, 22243, 19474, 19473, + 19479, 19471, 19483, 19470, 19477, 3256, 19536, 22242, 22045, 21764, 22082, 23273, 21882, 20103, 20107, 19520, 21884, + 20354, 20349, 20490, 20222, 20205, 20178, 20179, 20181, 20182, 20183, 20227, 24265}; public ToolboxDirectVideoExporter() { - System.out.println("Aloha, funky time"); + log.info("Aloha, funky time"); // proxy configuration if (proxy) { @@ -36,59 +99,88 @@ public class ToolboxDirectVideoExporter { config = RequestConfig.custom().setProxy(proxy).build(); } - httpPost = new HttpPost("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=20553"); - httpPost.setHeader("Cookie", cookie); - httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded"); - httpPost.setHeader("Referer", "https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=20553"); - - List urlParameters = new ArrayList(); -// urlParameters.add(new BasicNameValuePair("__EVENTTARGET", "")); -// urlParameters.add(new BasicNameValuePair("__EVENTARGUMENT", "")); - urlParameters.add(new BasicNameValuePair("__VIEWSTATE", "/wEPDwUIOTY3MTIxOTcPFggeB0Fzc2V0SUQCyaABHgFzZB4BbAUFZmFsc2UeDWludmVudG9yeUNvZGUFFUNSMDYxNUNSRUROWkExMzBWMDBSMBYCZg9kFgICAw9kFgQCBQ9kFhACAQ8WAh4HVmlzaWJsZWdkAgUPFgIeBWNsYXNzBQ5ob21lTG5rIGFjdGl2ZWQCBg8WAh8FBQtuYXRpb25hbExua2QCBw8WAh8FBQtkaXN0cmljdExua2QCCA8WAh8FBQhwaG90b0xua2QCCQ8WAh8FBQtleGNoYW5nZUxua2QCCg8WAh8FBQ1hZG1pbkxuayBsYXN0ZAILDxYCHwUFB2ZhdnNMbmtkAgcPZBYCAgMPZBYCZg9kFkRmDxYCHgRUZXh0BUg8aW1nIHNyYz0iL0JheWVyTUFNL2ltYWdlcy9pY29uX3ZpZGVvLmdpZiIgc3R5bGU9J3RleHQtYWxpZ246Q2VudGVyOycgLz5kAgYPDxYCHwYFIUNZMjAxNiBCYXllciBDcmVkZW56IEVkICYgRWRuYSBUVmRkAgcPFgIfBgUEMjAxNWQCCA8WAh8GBRBOYXRpb25hbENhbXBhaWduZAIJDxYCHwYFC05hdGlvbmFsIFRWZAIKDw8WAh8GBQdDcmVkZW56ZGQCCw8PFgIfBgUFQnJhbmRkZAIMDw8WAh8GBQdDcmVkZW56ZGQCDQ8PFgIfBgURQXJjaGl2ZWQgQnJhbmQocylkZAIODw8WAh8GZWRkAg8PFgIfBgUMQmVuICBDdXJ0aXMgZAIQDxYCHwYFCjAzLzA4LzIwMTdkAhEPFgIfBgUKMDMvMDgvMjAxN2QCEw8WAh8GBWpBZG1pbmlzdHJhdG9yLCBCQ1MgQWdlbmNpZXMsIEJpb1NjaWVuY2UsIENvcnBDb20sIENyb3AgUHJvdGVjdGlvbiwgR3VzdGFmc29uLCBNYXJDb20sIE90aGVyIEJDUywgT3V0U2lkZVVTZAIUDxYCHglpbm5lcmh0bWwFEEludmVudG9yeSBDb2RlOiBkAhUPFgIfBgUVQ1IwNjE1Q1JFRE5aQTEzMFYwMFIwZAIWDxYCHgVpbmRleAUFMjA1NTNkAhgPFgQfCAUFMjA1NTMfBGhkAhkPFgQfCAUFMjA1NTMeB0NoaWxkSUQFFU1haW5Db250ZW50X0Fzc2V0TmFtZWQCGg8WAh8EaGQCGw8WAh8IBQUyMDU1M2QCHA8PZBYCHgdvbmNsaWNrBRxyZXR1cm4gRGVsZXRlQ29uZmlybWF0aW9uKCk7ZAIeDw8WAh8EaGRkAiAPDxYCHwRnZGQCIQ8PFgIfBGdkZAIiDxYCHwZlZAIjDxYCHgtfIUl0ZW1Db3VudGZkAiUPEA8WBh4NRGF0YVRleHRGaWVsZAUFVmFsdWUeDkRhdGFWYWx1ZUZpZWxkBQNLZXkeC18hRGF0YUJvdW5kZ2QQFQENT3JpZ2luYWwgRmlsZRUBATEUKwMBZ2RkAiYPD2QWBB4DVGFnBSQzMTRlNjMyOC1jZWJhLTQ4NDMtOGI2NC03N2FiY2E0ZWQzMDEfCAUFMjA1NTNkAioPEA8WBh8MBQVWYWx1ZR8NBQNLZXkfDmdkEBUBDU9yaWdpbmFsIEZpbGUVAQExFCsDAWdkZAIrDw9kFggfCAUFMjA1NTMfCQUVTWFpbkNvbnRlbnRfQXNzZXROYW1lHghDaGlsZElEMQUYTWFpbkNvbnRlbnRfZGRsRENWZXJzaW9uHghDaGlsZElEMwUBNGQCNg8PZBYGHwkFfE1haW5Db250ZW50X3R4dEVtYWlsQWRkcmVzcyxNYWluQ29udGVudF90eHRNc2csTWFpbkNvbnRlbnRfcmRibGluayxNYWluQ29udGVudF9yZGJBdHRhY2htZW50LE1haW5Db250ZW50X2xibEVycm9yTXNnLDIwNTUzLDEfEAUWTWFpbkNvbnRlbnRfZXJyb3JFbWFpbB4IQ2hpbGRJRDIFGE1haW5Db250ZW50X2ltZ0VtYWlsV2FpdGQCOA8WAh8GBSFDWTIwMTYgQmF5ZXIgQ3JlZGVueiBFZCAmIEVkbmEgVFZkAjkPFgIfBgU+PGltZyBzcmM9Ii9CYXllck1BTS9pbWFnZXMvaWNvbl92aWRlby5naWYiIGNsYXNzPSJscmdJbWFnZSIgLz5kGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYDBRljdGwwMCRNYWluQ29udGVudCRyZGJsaW5rBR9jdGwwMCRNYWluQ29udGVudCRyZGJBdHRhY2htZW50BR9jdGwwMCRNYWluQ29udGVudCRyZGJBdHRhY2htZW50MOXQUOpfEQuKGLzy42k3Z/9E2uf6wnkLiqmAvV4ZdrI=")); - urlParameters.add(new BasicNameValuePair("__VIEWSTATEGENERATOR", "A7505276")); - urlParameters.add(new BasicNameValuePair("__EVENTVALIDATION", "/wEdAB6dDLGD3b7UVQWQvX2N+ThaW7jI95Exf+kRO8vblPx0OhA7PnnWXB5lbHzuNqY922eEBIKcA5vDAoqozy8UPbsP+CpAqXQu74+37bKN6ygCFXwIBlgB26GplwiKAIlODgDkhXUGebEfv2hj4J5JDcsReT3aut2MnjN793orsIjcA9/bvL4fWG7KRq+LDikVOFlEziFi1bGz4caWdQ0AEd956gcoNr6dPCe9l847FMZPfukXI4EP7/Vlz1KM+vPEqbR2oQ+0NDvJH4BbwBx9PY9VQN3lHbx0JN+PR1MESCvG5RmQXubDNN9zn2qvC6xP7zy0m7/nN0U29rlfplMBgB/YTSZhbmCNXWmOpMmroFz5uQCxOZp5/YzoW4qgaR4TVV6hBF1A3NfUwlnh7qnxUAEAQX9+ScDGDIM47zqhQCjs5YxCI9I8X1add3iKzB4uP17Q3RQl1cTJdiMKptKpB0qULiO62/d+SdcFLN3gB+pSfQkyDLJd2CG5K6KqwOlXZON5TuU0eHcd6HT4HyGWQhTxiJwiI7HoCHdMXxisdgDYwsycVXlLyuw1aRvn0qEPSa5cbjUzNSPCEiQI+S3hylsgYbXYLUXJMQ1MJOEgfJSs7oDU9mEigJlYs2X1yTq0lvL4N7DzzZ59A3GfpWf+JbGOJprNnly7H9Q9Sa5KlKc9PQ==")); - urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdncatID", "4")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdnName", "CY2016 Bayer Credenz Ed & Edna TV")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$ddlVersionsPopup", "1")); - urlParameters.add(new BasicNameValuePair("ctl00$MainContent$btnDownloadOk", "Ok")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$ddlDCVersion", "1")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$txtEmailAddress", "")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$mail", "rdblink")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$txtMsg", "")); -// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdnrevesionID", "1")); - - try { - httpPost.setEntity(new UrlEncodedFormEntity(urlParameters)); - } catch (UnsupportedEncodingException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - - if (proxy) - httpPost.setConfig(config); - - try { - CloseableHttpResponse response = httpClient.execute(target, httpPost); + for (int i = 0; i < assetId.length; i++) { + log.info("AssetID: " + assetId[i]); + String filePath = baseDir + assetId[i] + ".zip"; - System.out.println(EntityUtils.toString(response.getEntity())); + if (new File(filePath).exists()) { + log.info(filePath + " exists, skipping"); + break; + } -// HttpEntity entity = response.getEntity(); -// InputStream is = entity.getContent(); -// String filePath = "F:\\Toolbox\\Testing\\sample.zip"; -// FileOutputStream fos = new FileOutputStream(new File(filePath)); -// int inByte; -// while((inByte = is.read()) != -1) -// fos.write(inByte); -// is.close(); -// fos.close(); - } - catch (Exception e) { - e.printStackTrace(); + try { + httpGet = new HttpGet("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=" + assetId[i]); + httpGet.setHeader("Cookie", cookie); + if (proxy) httpGet.setConfig(config); + + response = httpClient.execute(target, httpGet); + + log.info(response.getStatusLine()); + HttpEntity entity = response.getEntity(); + String content = EntityUtils.toString(response.getEntity()); + int viewStateIndex = content.indexOf(viewStateMarker) + viewStateMarker.length(); + String viewState = content.substring(viewStateIndex, content.indexOf("\" />", viewStateIndex)); + int eventValidationIndex = content.indexOf(eventValidationMarker) + eventValidationMarker.length(); + String eventValidation = content.substring(eventValidationIndex, content.indexOf("\" />", eventValidationIndex)); + EntityUtils.consume(entity); + response.close(); + +// System.out.println("viewState: " + viewState); +// System.out.println("eventValidation: " + eventValidation); + + httpPost = new HttpPost("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=" + assetId[i]); + httpPost.setHeader("Cookie", cookie); + httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded"); + if (proxy) httpPost.setConfig(config); + + urlParameters = new ArrayList(); + urlParameters.add(new BasicNameValuePair("__VIEWSTATE", viewState)); + urlParameters.add(new BasicNameValuePair("__EVENTVALIDATION", eventValidation)); + urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdncatID", "4")); + urlParameters.add(new BasicNameValuePair("ctl00$MainContent$btnDownloadOk", "Ok")); + + httpPost.setEntity(new UrlEncodedFormEntity(urlParameters)); + + response = httpClient.execute(target, httpPost); + + entity = response.getEntity(); + InputStream is = entity.getContent(); + FileOutputStream fos = new FileOutputStream(new File(filePath)); + int inByte; + while((inByte = is.read()) != -1) + fos.write(inByte); + is.close(); + fos.close(); + + EntityUtils.consume(entity); + response.close(); + } + catch (Exception e) { + e.printStackTrace(); + log.error(e.getMessage()); + } } } public static void main(String[] args) { - new ToolboxDirectVideoExporter(); + try { + Layout layout = new PatternLayout("%p %m%n"); // "%d{ISO8601} - %p %m%n" + BasicConfigurator.configure(new ConsoleAppender(layout)); + BasicConfigurator.configure(new RollingFileAppender(layout, "F:\\Toolbox\\Movies\\download.log", true)); + Logger.getRootLogger().setLevel(Level.INFO); + + long startTime = System.currentTimeMillis(); + log.debug("Start: " + startTime); + + new ToolboxDirectVideoExporter(); + + long endTime = System.currentTimeMillis(); + log.debug("End: " + endTime); + log.debug("Duration: " + (endTime - startTime)); + } catch (Exception e) { + e.printStackTrace(); + log.error(e.getMessage()); + } } }