incl. File.exists() check
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
package com.bayer.edam;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -10,25 +12,86 @@ import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.log4j.BasicConfigurator;
|
||||
import org.apache.log4j.ConsoleAppender;
|
||||
import org.apache.log4j.Layout;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.apache.log4j.PatternLayout;
|
||||
import org.apache.log4j.RollingFileAppender;
|
||||
|
||||
public class ToolboxDirectVideoExporter {
|
||||
|
||||
private static Logger log = Logger.getLogger(ToolboxDirectVideoExporter.class);
|
||||
|
||||
String cookie = "ASP.NET_SessionId=kboqi1uznhyvxsgyezsl5z3t; userid=1041";
|
||||
boolean proxy = true;
|
||||
|
||||
static final String viewStateMarker = "id=\"__VIEWSTATE\" value=\"";
|
||||
static final String eventValidationMarker = "id=\"__EVENTVALIDATION\" value=\"";
|
||||
static final String baseDir = "F:\\Toolbox\\Movies\\";
|
||||
|
||||
CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
HttpHost target = new HttpHost("bayertoolbox.com", 443, "https");
|
||||
HttpPost httpPost;
|
||||
HttpGet httpGet;
|
||||
HttpEntity httpEntity;
|
||||
RequestConfig config;
|
||||
CloseableHttpResponse response;
|
||||
List<NameValuePair> urlParameters;
|
||||
|
||||
// int[] testId = {20553, 3439, 23445, 11428};
|
||||
|
||||
int[] assetId = {19664, 20553, 4060, 10042, 9659, 9660, 9658, 10041, 9652, 9656, 4135, 19332, 19488, 10483, 11924, 1392,
|
||||
11923, 10482, 8071, 1396, 10481, 11922, 3200, 11920, 10479, 20711, 10472, 11913, 10607, 3342, 9657, 3201, 11921,
|
||||
10480, 1382, 4046, 10478, 11919, 11350, 1393, 10355, 10474, 11915, 1395, 10490, 11788, 11789, 10491, 3204, 3202, 3338,
|
||||
15700, 11917, 10476, 10475, 11916, 10745, 10752, 10785, 13996, 11332, 13723, 11912, 10471, 20961, 13722, 11428, 15397,
|
||||
11353, 2396, 10762, 2398, 11787, 10489, 3431, 10473, 11914, 10606, 13459, 13719, 11772, 3184, 4264, 13758, 2234, 1435,
|
||||
11894, 11897, 3205, 11352, 9655, 13997, 3203, 20502, 11899, 13730, 10500, 11801, 2188, 11327, 1442, 2387, 4124, 4115,
|
||||
4239, 3198, 3199, 11336, 10551, 11335, 10292, 10550, 10553, 10291, 9653, 9654, 10295, 10545, 10286, 11807, 11804,
|
||||
11344, 11774, 20506, 11328, 2399, 1439, 11348, 13994, 14136, 3442, 11349, 11902, 11270, 22936, 11896, 1394, 20503,
|
||||
11949, 2395, 10495, 11793, 11900, 3434, 1039, 1467, 10608, 20504, 11785, 11786, 10488, 11287, 23289, 11895, 11893,
|
||||
11773, 20501, 19579, 23510, 1441, 22937, 23288, 23490, 14322, 13726, 20507, 11797, 11798, 10496, 13721, 20505, 3432,
|
||||
19382, 20200, 10356, 11951, 19491, 23233, 11117, 19498, 11790, 10492, 23643, 11824, 11898, 11802, 10501, 19493, 18843,
|
||||
3441, 4263, 23086, 10746, 4535, 11901, 11952, 19492, 19487, 22947, 23870, 23783, 19500, 11820, 23550, 19490, 10384,
|
||||
19495, 19489, 19494, 19496, 11918, 10477, 19486, 11347, 3339, 19499, 11285, 11346, 13995, 11821, 19917, 1443, 11950,
|
||||
13589, 3206, 23997, 11780, 11781, 10485, 11822, 14160, 13728, 11910, 23756, 20059, 23697, 11838, 14162, 14151, 13720,
|
||||
11827, 13991, 11962, 11823, 23839, 23209, 11830, 1438, 11826, 11967, 10605, 11825, 11828, 10487, 11783, 11784, 11337,
|
||||
11891, 23087, 11928, 23236, 11333, 11887, 11808, 11791, 10493, 11836, 11905, 11961, 10499, 11800, 11875, 11884, 13115,
|
||||
11528, 11829, 11832, 10761, 2386, 11286, 1440, 10294, 11926, 12950, 10554, 11946, 10296, 3440, 10549, 10290, 12948,
|
||||
11964, 11176, 10548, 10289, 21024, 11429, 10544, 11839, 10494, 11792, 10285, 14474, 21013, 11878, 11831, 3435, 11873,
|
||||
14265, 2393, 11909, 20731, 11782, 10486, 13731, 11338, 11835, 11879, 11965, 11943, 19329, 2397, 11345, 14161, 11968,
|
||||
2392, 11956, 11953, 3337, 11889, 13458, 11813, 11882, 11906, 11954, 11963, 11958, 11834, 3433, 11970, 14321, 13467,
|
||||
13468, 11343, 11840, 11837, 11957, 10497, 11890, 11871, 1380, 11799, 10498, 11118, 2480, 2494, 13725, 11810, 11811,
|
||||
11812, 13729, 10357, 11904, 3439, 11969, 13471, 11945, 11877, 11929, 14589, 11886, 11888, 11960, 11814, 11966, 11326,
|
||||
10760, 21266, 10361, 11941, 11883, 11959, 20545, 11324, 11325, 11179, 20546, 20421, 1437, 11334, 20422, 11927, 11876,
|
||||
20484, 11341, 23617, 11880, 23445, 23446, 3437, 11870, 11938, 11364, 11872, 11944, 11815, 11925, 11833, 11881, 11885,
|
||||
11809, 19538, 19919, 11331, 19934, 18793, 3438, 11935, 11351, 11942, 12141, 11329, 11818, 4134, 11936, 11330, 11930,
|
||||
11340, 3347, 12162, 11908, 11937, 11816, 11940, 11932, 11819, 11874, 11869, 11971, 19060, 11933, 11750, 13590, 12403,
|
||||
12215, 19497, 11939, 22148, 3368, 23683, 13933, 14166, 3436, 14152, 14163, 11342, 11817, 23716, 10383, 11907, 23615,
|
||||
13470, 23714, 11972, 19333, 20668, 23682, 13469, 11903, 21396, 23613, 23612, 23718, 11934, 23836, 23497, 2391, 23713,
|
||||
11973, 2078, 14159, 13727, 23499, 23829, 23500, 10205, 23616, 2394, 10108, 14900, 2385, 20938, 14409, 23181, 23439,
|
||||
23831, 23840, 21006, 3430, 23832, 23837, 14191, 14907, 23868, 23370, 23614, 11931, 23833, 23830, 23841, 3134, 19704,
|
||||
19918, 23838, 13992, 23834, 13513, 19576, 13113, 13993, 13512, 2382, 19922, 19920, 19921, 2383, 19650, 11339, 23681,
|
||||
18724, 14411, 8893, 23712, 15288, 23314, 23024, 14412, 21014, 19963, 23717, 19480, 23715, 23372, 8815, 14410, 19832,
|
||||
18569, 18722, 19787, 10987, 10988, 11050, 2600, 21335, 21347, 21302, 12949, 18721, 18568, 13426, 13694, 13724, 11432,
|
||||
11361, 11436, 11435, 11173, 11433, 11434, 11431, 11178, 13511, 11175, 13509, 20960, 21303, 13427, 13425, 20937, 10034,
|
||||
15320, 22171, 20972, 10359, 13424, 18723, 20555, 13423, 11097, 13361, 11753, 13305, 19482, 19476, 23279, 23280, 10353,
|
||||
23089, 24418, 2079, 10354, 23281, 10360, 23276, 10148, 24417, 23048, 24419, 12868, 10149, 13306, 13275, 10358, 13301,
|
||||
21235, 12805, 13304, 22959, 11048, 1041, 20635, 20636, 23546, 24420, 12585, 19277, 3984, 3491, 24274, 2401, 22525,
|
||||
13323, 8803, 23648, 10546, 10287, 21900, 23680, 2402, 23633, 22646, 21899, 24276, 11446, 23795, 985, 2390, 24275,
|
||||
22464, 23610, 21883, 1040, 10609, 23474, 23746, 19475, 19484, 22046, 19472, 19481, 22047, 19478, 22243, 19474, 19473,
|
||||
19479, 19471, 19483, 19470, 19477, 3256, 19536, 22242, 22045, 21764, 22082, 23273, 21882, 20103, 20107, 19520, 21884,
|
||||
20354, 20349, 20490, 20222, 20205, 20178, 20179, 20181, 20182, 20183, 20227, 24265};
|
||||
|
||||
public ToolboxDirectVideoExporter() {
|
||||
System.out.println("Aloha, funky time");
|
||||
log.info("Aloha, funky time");
|
||||
|
||||
// proxy configuration
|
||||
if (proxy) {
|
||||
@@ -36,59 +99,88 @@ public class ToolboxDirectVideoExporter {
|
||||
config = RequestConfig.custom().setProxy(proxy).build();
|
||||
}
|
||||
|
||||
httpPost = new HttpPost("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=20553");
|
||||
httpPost.setHeader("Cookie", cookie);
|
||||
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
|
||||
httpPost.setHeader("Referer", "https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=20553");
|
||||
for (int i = 0; i < assetId.length; i++) {
|
||||
log.info("AssetID: " + assetId[i]);
|
||||
String filePath = baseDir + assetId[i] + ".zip";
|
||||
|
||||
List<NameValuePair> urlParameters = new ArrayList<NameValuePair>();
|
||||
// urlParameters.add(new BasicNameValuePair("__EVENTTARGET", ""));
|
||||
// urlParameters.add(new BasicNameValuePair("__EVENTARGUMENT", ""));
|
||||
urlParameters.add(new BasicNameValuePair("__VIEWSTATE", "/wEPDwUIOTY3MTIxOTcPFggeB0Fzc2V0SUQCyaABHgFzZB4BbAUFZmFsc2UeDWludmVudG9yeUNvZGUFFUNSMDYxNUNSRUROWkExMzBWMDBSMBYCZg9kFgICAw9kFgQCBQ9kFhACAQ8WAh4HVmlzaWJsZWdkAgUPFgIeBWNsYXNzBQ5ob21lTG5rIGFjdGl2ZWQCBg8WAh8FBQtuYXRpb25hbExua2QCBw8WAh8FBQtkaXN0cmljdExua2QCCA8WAh8FBQhwaG90b0xua2QCCQ8WAh8FBQtleGNoYW5nZUxua2QCCg8WAh8FBQ1hZG1pbkxuayBsYXN0ZAILDxYCHwUFB2ZhdnNMbmtkAgcPZBYCAgMPZBYCZg9kFkRmDxYCHgRUZXh0BUg8aW1nIHNyYz0iL0JheWVyTUFNL2ltYWdlcy9pY29uX3ZpZGVvLmdpZiIgc3R5bGU9J3RleHQtYWxpZ246Q2VudGVyOycgLz5kAgYPDxYCHwYFIUNZMjAxNiBCYXllciBDcmVkZW56IEVkICYgRWRuYSBUVmRkAgcPFgIfBgUEMjAxNWQCCA8WAh8GBRBOYXRpb25hbENhbXBhaWduZAIJDxYCHwYFC05hdGlvbmFsIFRWZAIKDw8WAh8GBQdDcmVkZW56ZGQCCw8PFgIfBgUFQnJhbmRkZAIMDw8WAh8GBQdDcmVkZW56ZGQCDQ8PFgIfBgURQXJjaGl2ZWQgQnJhbmQocylkZAIODw8WAh8GZWRkAg8PFgIfBgUMQmVuICBDdXJ0aXMgZAIQDxYCHwYFCjAzLzA4LzIwMTdkAhEPFgIfBgUKMDMvMDgvMjAxN2QCEw8WAh8GBWpBZG1pbmlzdHJhdG9yLCBCQ1MgQWdlbmNpZXMsIEJpb1NjaWVuY2UsIENvcnBDb20sIENyb3AgUHJvdGVjdGlvbiwgR3VzdGFmc29uLCBNYXJDb20sIE90aGVyIEJDUywgT3V0U2lkZVVTZAIUDxYCHglpbm5lcmh0bWwFEEludmVudG9yeSBDb2RlOiBkAhUPFgIfBgUVQ1IwNjE1Q1JFRE5aQTEzMFYwMFIwZAIWDxYCHgVpbmRleAUFMjA1NTNkAhgPFgQfCAUFMjA1NTMfBGhkAhkPFgQfCAUFMjA1NTMeB0NoaWxkSUQFFU1haW5Db250ZW50X0Fzc2V0TmFtZWQCGg8WAh8EaGQCGw8WAh8IBQUyMDU1M2QCHA8PZBYCHgdvbmNsaWNrBRxyZXR1cm4gRGVsZXRlQ29uZmlybWF0aW9uKCk7ZAIeDw8WAh8EaGRkAiAPDxYCHwRnZGQCIQ8PFgIfBGdkZAIiDxYCHwZlZAIjDxYCHgtfIUl0ZW1Db3VudGZkAiUPEA8WBh4NRGF0YVRleHRGaWVsZAUFVmFsdWUeDkRhdGFWYWx1ZUZpZWxkBQNLZXkeC18hRGF0YUJvdW5kZ2QQFQENT3JpZ2luYWwgRmlsZRUBATEUKwMBZ2RkAiYPD2QWBB4DVGFnBSQzMTRlNjMyOC1jZWJhLTQ4NDMtOGI2NC03N2FiY2E0ZWQzMDEfCAUFMjA1NTNkAioPEA8WBh8MBQVWYWx1ZR8NBQNLZXkfDmdkEBUBDU9yaWdpbmFsIEZpbGUVAQExFCsDAWdkZAIrDw9kFggfCAUFMjA1NTMfCQUVTWFpbkNvbnRlbnRfQXNzZXROYW1lHghDaGlsZElEMQUYTWFpbkNvbnRlbnRfZGRsRENWZXJzaW9uHghDaGlsZElEMwUBNGQCNg8PZBYGHwkFfE1haW5Db250ZW50X3R4dEVtYWlsQWRkcmVzcyxNYWluQ29udGVudF90eHRNc2csTWFpbkNvbnRlbnRfcmRibGluayxNYWluQ29udGVudF9yZGJBdHRhY2htZW50LE1haW5Db250ZW50X2xibEVycm9yTXNnLDIwNTUzLDEfEAUWTWFpbkNvbnRlbnRfZXJyb3JFbWFpbB4IQ2hpbGRJRDIFGE1haW5Db250ZW50X2ltZ0VtYWlsV2FpdGQCOA8WAh8GBSFDWTIwMTYgQmF5ZXIgQ3JlZGVueiBFZCAmIEVkbmEgVFZkAjkPFgIfBgU+PGltZyBzcmM9Ii9CYXllck1BTS9pbWFnZXMvaWNvbl92aWRlby5naWYiIGNsYXNzPSJscmdJbWFnZSIgLz5kGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYDBRljdGwwMCRNYWluQ29udGVudCRyZGJsaW5rBR9jdGwwMCRNYWluQ29udGVudCRyZGJBdHRhY2htZW50BR9jdGwwMCRNYWluQ29udGVudCRyZGJBdHRhY2htZW50MOXQUOpfEQuKGLzy42k3Z/9E2uf6wnkLiqmAvV4ZdrI="));
|
||||
urlParameters.add(new BasicNameValuePair("__VIEWSTATEGENERATOR", "A7505276"));
|
||||
urlParameters.add(new BasicNameValuePair("__EVENTVALIDATION", "/wEdAB6dDLGD3b7UVQWQvX2N+ThaW7jI95Exf+kRO8vblPx0OhA7PnnWXB5lbHzuNqY922eEBIKcA5vDAoqozy8UPbsP+CpAqXQu74+37bKN6ygCFXwIBlgB26GplwiKAIlODgDkhXUGebEfv2hj4J5JDcsReT3aut2MnjN793orsIjcA9/bvL4fWG7KRq+LDikVOFlEziFi1bGz4caWdQ0AEd956gcoNr6dPCe9l847FMZPfukXI4EP7/Vlz1KM+vPEqbR2oQ+0NDvJH4BbwBx9PY9VQN3lHbx0JN+PR1MESCvG5RmQXubDNN9zn2qvC6xP7zy0m7/nN0U29rlfplMBgB/YTSZhbmCNXWmOpMmroFz5uQCxOZp5/YzoW4qgaR4TVV6hBF1A3NfUwlnh7qnxUAEAQX9+ScDGDIM47zqhQCjs5YxCI9I8X1add3iKzB4uP17Q3RQl1cTJdiMKptKpB0qULiO62/d+SdcFLN3gB+pSfQkyDLJd2CG5K6KqwOlXZON5TuU0eHcd6HT4HyGWQhTxiJwiI7HoCHdMXxisdgDYwsycVXlLyuw1aRvn0qEPSa5cbjUzNSPCEiQI+S3hylsgYbXYLUXJMQ1MJOEgfJSs7oDU9mEigJlYs2X1yTq0lvL4N7DzzZ59A3GfpWf+JbGOJprNnly7H9Q9Sa5KlKc9PQ=="));
|
||||
urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdncatID", "4"));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdnName", "CY2016 Bayer Credenz Ed & Edna TV"));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$ddlVersionsPopup", "1"));
|
||||
urlParameters.add(new BasicNameValuePair("ctl00$MainContent$btnDownloadOk", "Ok"));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$ddlDCVersion", "1"));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$txtEmailAddress", ""));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$mail", "rdblink"));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$txtMsg", ""));
|
||||
// urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdnrevesionID", "1"));
|
||||
|
||||
try {
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(urlParameters));
|
||||
} catch (UnsupportedEncodingException e1) {
|
||||
// TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
if (new File(filePath).exists()) {
|
||||
log.info(filePath + " exists, skipping");
|
||||
break;
|
||||
}
|
||||
|
||||
if (proxy)
|
||||
httpPost.setConfig(config);
|
||||
|
||||
try {
|
||||
CloseableHttpResponse response = httpClient.execute(target, httpPost);
|
||||
httpGet = new HttpGet("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=" + assetId[i]);
|
||||
httpGet.setHeader("Cookie", cookie);
|
||||
if (proxy) httpGet.setConfig(config);
|
||||
|
||||
System.out.println(EntityUtils.toString(response.getEntity()));
|
||||
response = httpClient.execute(target, httpGet);
|
||||
|
||||
// HttpEntity entity = response.getEntity();
|
||||
// InputStream is = entity.getContent();
|
||||
// String filePath = "F:\\Toolbox\\Testing\\sample.zip";
|
||||
// FileOutputStream fos = new FileOutputStream(new File(filePath));
|
||||
// int inByte;
|
||||
// while((inByte = is.read()) != -1)
|
||||
// fos.write(inByte);
|
||||
// is.close();
|
||||
// fos.close();
|
||||
log.info(response.getStatusLine());
|
||||
HttpEntity entity = response.getEntity();
|
||||
String content = EntityUtils.toString(response.getEntity());
|
||||
int viewStateIndex = content.indexOf(viewStateMarker) + viewStateMarker.length();
|
||||
String viewState = content.substring(viewStateIndex, content.indexOf("\" />", viewStateIndex));
|
||||
int eventValidationIndex = content.indexOf(eventValidationMarker) + eventValidationMarker.length();
|
||||
String eventValidation = content.substring(eventValidationIndex, content.indexOf("\" />", eventValidationIndex));
|
||||
EntityUtils.consume(entity);
|
||||
response.close();
|
||||
|
||||
// System.out.println("viewState: " + viewState);
|
||||
// System.out.println("eventValidation: " + eventValidation);
|
||||
|
||||
httpPost = new HttpPost("https://bayertoolbox.com/Assets/AssetDetailNational.aspx?AssetID=" + assetId[i]);
|
||||
httpPost.setHeader("Cookie", cookie);
|
||||
httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
|
||||
if (proxy) httpPost.setConfig(config);
|
||||
|
||||
urlParameters = new ArrayList<NameValuePair>();
|
||||
urlParameters.add(new BasicNameValuePair("__VIEWSTATE", viewState));
|
||||
urlParameters.add(new BasicNameValuePair("__EVENTVALIDATION", eventValidation));
|
||||
urlParameters.add(new BasicNameValuePair("ctl00$MainContent$hdncatID", "4"));
|
||||
urlParameters.add(new BasicNameValuePair("ctl00$MainContent$btnDownloadOk", "Ok"));
|
||||
|
||||
httpPost.setEntity(new UrlEncodedFormEntity(urlParameters));
|
||||
|
||||
response = httpClient.execute(target, httpPost);
|
||||
|
||||
entity = response.getEntity();
|
||||
InputStream is = entity.getContent();
|
||||
FileOutputStream fos = new FileOutputStream(new File(filePath));
|
||||
int inByte;
|
||||
while((inByte = is.read()) != -1)
|
||||
fos.write(inByte);
|
||||
is.close();
|
||||
fos.close();
|
||||
|
||||
EntityUtils.consume(entity);
|
||||
response.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
Layout layout = new PatternLayout("%p %m%n"); // "%d{ISO8601} - %p %m%n"
|
||||
BasicConfigurator.configure(new ConsoleAppender(layout));
|
||||
BasicConfigurator.configure(new RollingFileAppender(layout, "F:\\Toolbox\\Movies\\download.log", true));
|
||||
Logger.getRootLogger().setLevel(Level.INFO);
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
log.debug("Start: " + startTime);
|
||||
|
||||
new ToolboxDirectVideoExporter();
|
||||
|
||||
long endTime = System.currentTimeMillis();
|
||||
log.debug("End: " + endTime);
|
||||
log.debug("Duration: " + (endTime - startTime));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user