tilman/ah-treereader
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

maintenance and 2nd run

Browse Source
This commit is contained in:
tliero
2017-05-15 08:36:34 +02:00
parent 8f6fc214cd
commit b1038058cb
4 changed files with 1050 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AH-TreeReader/2017-05-15_082639.log
+1037
View File
File diff suppressed because it is too large Load Diff
AH-TreeReader/src/de/kompf/javaxml/XPathReader.java
-193
View File
@@ -1,193 +0,0 @@
package de.kompf.javaxml;
import java.io.*;
import java.net.URL;
import java.util.*;
import javax.xml.XMLConstants;
import javax.xml.namespace.NamespaceContext;
import javax.xml.parsers.*;
import javax.xml.xpath.*;
import org.w3c.dom.*;
import org.xml.sax.SAXException;
/**
* Sample code how to use to XPath API to extract information from XML data.
*
* @author Kompf
*
*/
public class XPathReader {
/**
* Evaluate an XML input stream using the given xpath.
*
* @param in The XML input stream.
* @param xpathExpr The xpath expression - must not contain any namespace
* prefixes.
* @param result A collection to append the results to.
*/
void eval(InputStream in, String xpathExpr, Collection<String> result)
throws ParserConfigurationException, SAXException, IOException,
XPathExpressionException {
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(false); // important!
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
Document doc = docBuilder.parse(in);
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression expr = xpath.compile(xpathExpr);
NodeList nodeList = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
for (int i = 0; i < nodeList.getLength(); ++i) {
Node node = nodeList.item(i);
result.add(node.getNodeValue());
}
}
/**
* Evaluate an XML input stream using the given xpath. This implementation is
* namespace aware.
*
* @param in The XML input stream.
* @param xpathExpr The xpath expression - may contain namespace prefixes.
* @param nsCtx The namespace context to resolve the namespace prefixes from
* the xpath expression.
* @param result A collection to append the results to.
*/
void evalNamespaceAware(InputStream in, String xpathExpr,
NamespaceContext nsCtx, Collection<String> result)
throws ParserConfigurationException, SAXException, IOException,
XPathExpressionException {
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
docFactory.setNamespaceAware(true); // important!
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
Document doc = docBuilder.parse(in);
XPath xpath = XPathFactory.newInstance().newXPath();
xpath.setNamespaceContext(nsCtx);
XPathExpression expr = xpath.compile(xpathExpr);
NodeList nodeList = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
for (int i = 0; i < nodeList.getLength(); ++i) {
Node node = nodeList.item(i);
result.add(node.getNodeValue());
}
}
/**
* Print the result.
*
* @param result The collection of results.
* @param out The print stream to use.
*/
void printResult(Collection<String> result, PrintStream out) {
// print result
for (String name : result) {
out.println(name);
}
}
/**
* Read the titles of entries from the Heise news feed.
*/
void readHeiseFeed() throws Exception {
URL heiseFeed = new URL("http://www.heise.de/newsticker/heise-atom.xml");
InputStream in = heiseFeed.openStream();
Collection<String> result = new LinkedList<String>();
eval(in, "//entry/title/text()", result);
printResult(result, System.out);
}
/**
* Read the titles of entries from the Heise news feed. This is basically the
* same like {@link #readHeiseFeed()} but is aware of the namespace of the
* atom feed.
*/
void readHeiseFeedNamespaceAware() throws Exception {
URL heiseFeed = new URL("http://www.heise.de/newsticker/heise-atom.xml");
InputStream in = heiseFeed.openStream();
Collection<String> result = new LinkedList<String>();
evalNamespaceAware(in, "//ns:entry/ns:title/text()",
new SimpleNamespaceContext("ns", "http://www.w3.org/2005/Atom"), result);
printResult(result, System.out);
}
/**
* Read the titles of entries from the Twitter public time line. This is
* basically the same like {@link #readHeiseFeed()} but uses another URL.
*/
void readTwitterPublicTimeLine() throws Exception {
URL twitterFeed = new URL(
"http://api.twitter.com/1/statuses/public_timeline.atom");
InputStream in = twitterFeed.openStream();
Collection<String> result = new LinkedList<String>();
eval(in, "//entry/title/text()", result);
printResult(result, System.out);
}
/**
* Read all all names of 'way' elements from an OSM XML file. The names are
* kept sorted and unique using a TreeSet.
*/
void readOsmWayNames() throws Exception {
// OSM path names
URL osmUrl = new File("md.osm.xml").toURI().toURL();
InputStream in = osmUrl.openStream();
Collection<String> result = new TreeSet<String>();
eval(in, "/osm/way/tag[@k='name']/@v", result);
printResult(result, System.out);
}
/**
* MAIN.
*
* @param args ignored.
* @throws Exception If an error occurs.
*/
public static void main(String[] args) throws Exception {
XPathReader xPathReader = new XPathReader();
// Heise News
xPathReader.readHeiseFeed();
// Heise News namespace aware
//xPathReader.readHeiseFeedNamespaceAware();
// Twitter public time line
//xPathReader.readTwitterPublicTimeLine();
// OSM path names
//xPathReader.readOsmWayNames();
}
static class SimpleNamespaceContext implements NamespaceContext {
private String prefix;
private String uri;
public SimpleNamespaceContext(String prefix, String uri) {
this.prefix = prefix;
this.uri = uri;
}
public String getNamespaceURI(String prefix) {
if (this.prefix.equals(prefix)) {
return uri;
}
return XMLConstants.NULL_NS_URI;
}
public String getPrefix(String namespaceURI) {
if (uri.equals(namespaceURI)) {
return prefix;
}
return null;
}
@SuppressWarnings("unchecked")
public Iterator getPrefixes(String namespaceURI) {
List<String> prefixList = new ArrayList<String>();
if (uri.equals(namespaceURI)) {
prefixList.add(prefix);
}
return prefixList.iterator();
}
}
}
AH-TreeReader/src/de/tilman/AHTreeReader/AHTreeReader.java
+12 -5
View File
@@ -14,6 +14,7 @@ import javax.xml.xpath.XPathFactory;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.MultipartEntityBuilder;
@@ -27,13 +28,16 @@ import de.tilman.AHTreeReader.Tree.TreeNode;
public class AHTreeReader { public class AHTreeReader {
String cookie = "PHPSESSID=5nccupenl4b7tps9048bge76h3"; String cookie = "PHPSESSID=8eujq15omh9nq3kkkf5b97osp2";
boolean proxy = true;
Tree<String, String> tree = new Tree<String, String>(); Tree<String, String> tree = new Tree<String, String>();
CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
HttpHost target = new HttpHost("media-assistant.animalhealth.bayer.com", 443, "https"); HttpHost target = new HttpHost("media-assistant.animalhealth.bayer.com", 443, "https");
HttpPost httpPost; HttpPost httpPost;
HttpEntity httpEntity; HttpEntity httpEntity;
RequestConfig config;
DocumentBuilderFactory factory; DocumentBuilderFactory factory;
DocumentBuilder builder; DocumentBuilder builder;
@@ -45,11 +49,10 @@ public class AHTreeReader {
System.out.println("Aloha, funky time"); System.out.println("Aloha, funky time");
// proxy configuration // proxy configuration
/* if (proxy) {
HttpHost proxy = new HttpHost("10.185.190.100", 8080, "http"); HttpHost proxy = new HttpHost("10.185.190.100", 8080, "http");
RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); config = RequestConfig.custom().setProxy(proxy).build();
httpPost.setConfig(config); }
*/
try { try {
factory = DocumentBuilderFactory.newInstance(); factory = DocumentBuilderFactory.newInstance();
@@ -79,6 +82,8 @@ public class AHTreeReader {
// 2. Retrieve children and recurse // 2. Retrieve children and recurse
httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_getCategoryFolderSubnavi.php"); httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_getCategoryFolderSubnavi.php");
httpPost.setHeader("Cookie", cookie); httpPost.setHeader("Cookie", cookie);
if (proxy)
httpPost.setConfig(config);
// parent node is transferred as form data o_0 // parent node is transferred as form data o_0
httpEntity = MultipartEntityBuilder.create().addTextBody("parent", n.key).build(); httpEntity = MultipartEntityBuilder.create().addTextBody("parent", n.key).build();
@@ -122,6 +127,8 @@ public class AHTreeReader {
httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_loadCategoryTree.php"); httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_loadCategoryTree.php");
httpPost.setHeader("Cookie", cookie); httpPost.setHeader("Cookie", cookie);
if (proxy)
httpPost.setConfig(config);
CloseableHttpResponse response = httpClient.execute(target, httpPost); CloseableHttpResponse response = httpClient.execute(target, httpPost);
AH-TreeReader/src/de/tilman/AHTreeReader/PlainApp.java
-99
View File
@@ -1,99 +0,0 @@
package de.tilman.AHTreeReader;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import de.tilman.AHTreeReader.Tree.TreeNode;
public class PlainApp {
public static void main(String[] args) throws Exception {
System.out.println("Aloha, funky time");
String cookie = "PHPSESSID=5nccupenl4b7tps9048bge76h3";
Tree<String, String> tr = new Tree<String, String>();
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_loadCategoryTree.php");
HttpHost target = new HttpHost("media-assistant.animalhealth.bayer.com", 443, "https");
HttpHost proxy = new HttpHost("10.185.190.100", 8080, "http");
// RequestConfig config = RequestConfig.custom().setProxy(proxy).build();
// httpPost.setConfig(config);
httpPost.setHeader("Cookie", cookie);
HttpEntity entity = MultipartEntityBuilder.create().addTextBody("parent", "14344").build();
httpPost.setEntity(entity);
CloseableHttpResponse response = httpClient.execute(target, httpPost);
try {
System.out.println(response.getStatusLine());
String str = EntityUtils.toString(response.getEntity());
int startIndex = str.indexOf("</ul>");
String mainCategories = str.substring(str.indexOf("<ul>", startIndex)+5, str.indexOf("</ul>", startIndex + 5));
while (mainCategories.length() > 6) {
mainCategories = mainCategories.substring(mainCategories.indexOf("catId=") + 7);
String key = mainCategories.substring(0, mainCategories.indexOf('"'));
mainCategories = mainCategories.substring(mainCategories.indexOf(":void(0)'>") + 10);
String value = mainCategories.substring(0, mainCategories.indexOf('<'));
mainCategories = mainCategories.substring(mainCategories.indexOf('\n') + 1);
mainCategories = mainCategories.substring(mainCategories.indexOf('\n') + 1);
System.out.println(key + ", " + value);
tr.root.children.add(new TreeNode<String, String>(key, value, 0));
}
for (TreeNode<String, String> n : tr.root.children) {
httpPost = new HttpPost("https://media-assistant.animalhealth.bayer.com/assets/php/ajax_getCategoryFolderSubnavi.php");
// httpPost.setConfig(config);
httpPost.setHeader("Cookie", cookie);
entity = MultipartEntityBuilder.create().addTextBody("parent", n.key).build();
httpPost.setEntity(entity);
response = httpClient.execute(target, httpPost);
System.out.println("\n\n ~~~~~~~~~~~~~");
System.out.println(response.getStatusLine());
str = EntityUtils.toString(response.getEntity());
if (str.indexOf("<ul>") >= 0) {
}
}
} finally {
response.close();
}
// ArrayList<NameValuePair> postParameters;
//
// postParameters = new ArrayList<NameValuePair>();
// postParameters.add(new BasicNameValuePair("param1", "param1_value"));
// postParameters.add(new BasicNameValuePair("param2", "param2_value"));
//
// httpPost.setEntity(new UrlEncodedFormEntity(postParameters));
//
// HttpResponse response = httpclient.execute(httpPost);
}
}