Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
package com.stackexchange.toolbox.misc;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
// cf. https://meta.stackexchange.com/a/341993/295232
// and https://stackapps.com/a/8547/34061
public class SEDEClient {
public static void main(String[] args) throws Exception {
final CloseableHttpClient client = HttpClients.createDefault();
// Start OAuth2 session
HttpPost postRequest = new HttpPost("https://data.stackexchange.com/user/authenticate");
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("oauth2url", "https://stackoverflow.com/oauth"));
params.add(new BasicNameValuePair("openid", ""));
postRequest.setEntity(new UrlEncodedFormEntity(params));
String loginURL;
try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
loginURL = httpResponse.getFirstHeader("Location").getValue();
}
// We need to read and set some cookies in between redirections
RequestConfig noRedirectRequestConfig = RequestConfig.custom().setRedirectsEnabled(false).build();
// Providence cookie
String providenceCookie = null;
HttpGet getRequest = new HttpGet(loginURL);
getRequest.setConfig(noRedirectRequestConfig);
try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
loginURL = new URL(new URL(loginURL), httpResponse.getFirstHeader("Location").getValue()).toString();
for (Header header : httpResponse.getHeaders("Set-Cookie")) {
String cookie = header.getValue().split(";")[0];
if (cookie.startsWith("prov=")) {
providenceCookie = cookie;
break;
}
}
}
if (providenceCookie == null) {
throw new Exception("No providence cookie found.");
}
// Load login form
getRequest = new HttpGet(loginURL);
getRequest.setHeader("Cookie", providenceCookie);
String fkey, loginActionURL;
try (CloseableHttpResponse httpResponse = client.execute(getRequest);
InputStream inputStream = httpResponse.getEntity().getContent()) {
Document document = Jsoup.parse(inputStream, "UTF-8", loginURL);
loginActionURL = document.getElementById("login-form").absUrl("action");
fkey = document.selectFirst("input[name='fkey']").val();
}
// Submit login form
postRequest = new HttpPost(loginActionURL);
postRequest.setHeader("Cookie", providenceCookie);
params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("fkey", fkey));
params.add(new BasicNameValuePair("email", EMAIL));
params.add(new BasicNameValuePair("password", PASSWORD));
postRequest.setEntity(new UrlEncodedFormEntity(params));
String redirectURL, accountCookie = null;
try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
redirectURL = httpResponse.getFirstHeader("Location").getValue();
for (Header header : httpResponse.getHeaders("Set-Cookie")) {
String cookie = header.getValue().split(";")[0];
if (cookie.startsWith("acct=")) {
accountCookie = cookie;
break;
}
}
}
if (accountCookie == null) {
throw new Exception("No account cookie found.");
}
// OAuth2 redirection
String location = redirectURL, authenticationCookie = null;
// 1. https://stackoverflow.com/oauth?client_id=...
// 2. https://data.stackexchange.com/user/oauth/stackapps?code=...
// => /
for (int i = 1; i <= 2; i++) {
// the state parameter contains JSON, and Java doesn't like that unless it's %-encoded:
URL url = new URL(URLDecoder.decode(location, "UTF-8"));
URI uri = new URI(url.getProtocol(), url.getUserInfo(), url.getHost(), url.getPort(), url.getPath(), url.getQuery(), url.getRef());
getRequest = new HttpGet(uri);
getRequest.addHeader("Cookie", providenceCookie + (i == 1 ? "; " + accountCookie : ""));
getRequest.setConfig(noRedirectRequestConfig);
try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
location = httpResponse.getFirstHeader("Location").getValue();
System.out.println(location);
if (i == 2) {
for (Header header : httpResponse.getHeaders("Set-Cookie")) {
String cookie = header.getValue().split(";")[0];
if (cookie.startsWith(".ASPXAUTH=")) {
authenticationCookie = cookie;
break;
}
}
}
}
}
if (authenticationCookie == null) {
throw new Exception("No authentication cookie found.");
}
// Execute query
int siteID = 1; // 1 = Stack Overflow; see https://data.stackexchange.com/sites
int queryID = 748298, revisionID = 1445524;
postRequest = new HttpPost(
"https://data.stackexchange.com/query/run/" + siteID + "/" + queryID + "/" + revisionID);
params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("badgename", "Legendary"));
postRequest.setEntity(new UrlEncodedFormEntity(params));
JSONObject jRoot;
String jobID;
try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
jRoot = new JSONObject(IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8"));
jobID = jRoot.has("job_id") ? jRoot.getString("job_id") : null;
if (jobID != null) {
// Results not in cache yet
System.out.println("Starting job: " + jobID);
}
}
while (jRoot.optBoolean("running")) {
Thread.sleep(1000);
// Poll job
getRequest = new HttpGet(
"https://data.stackexchange.com/query/job/" + jobID + "?_=" + new Date().getTime());
try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
jRoot = new JSONObject(IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8"));
}
}
// Process results
if (jRoot.has("resultSets")) {
String messages = jRoot.getString("messages");
int totalResults = jRoot.getInt("totalResults"); // NOTE: only populated when the results weren't cached
int executionTime = jRoot.getInt("executionTime");
JSONArray jResultSets = jRoot.getJSONArray("resultSets");
for (int i = 0; i < jResultSets.length(); i++) {
JSONObject jResultSet = jResultSets.getJSONObject(i);
JSONArray jColumns = jResultSet.getJSONArray("columns");
for (int j = 0; j < jColumns.length(); j++) {
JSONObject jColumn = jColumns.getJSONObject(j);
String name = jColumn.getString("name"), type = jColumn.getString("type");
}
JSONArray jRows = jResultSet.getJSONArray("rows");
for (int j = 0; j < jRows.length(); j++) {
JSONArray jRow = jRows.getJSONArray(j);
}
}
} else {
String error = jRoot.has("error") ? jRoot.getString("error") : null;
}
System.out.println(jRoot);
System.exit(0);
}
private static final String EMAIL = "info@example.com", PASSWORD = "secret";
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment