SEDEClient.java

package com.stackexchange.toolbox.misc;

import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

// cf. https://meta.stackexchange.com/a/341993/295232
// and https://stackapps.com/a/8547/34061
public class SEDEClient {
	public static void main(String[] args) throws Exception {
		final CloseableHttpClient client = HttpClients.createDefault();

		// Start OAuth2 session
		HttpPost postRequest = new HttpPost("https://data.stackexchange.com/user/authenticate");
		List<NameValuePair> params = new ArrayList<NameValuePair>();
		params.add(new BasicNameValuePair("oauth2url", "https://stackoverflow.com/oauth"));
		params.add(new BasicNameValuePair("openid", ""));
		postRequest.setEntity(new UrlEncodedFormEntity(params));
		String loginURL;
		try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
			loginURL = httpResponse.getFirstHeader("Location").getValue();
		}

		// We need to read and set some cookies in between redirections
		RequestConfig noRedirectRequestConfig = RequestConfig.custom().setRedirectsEnabled(false).build();

		// Providence cookie
		String providenceCookie = null;
		HttpGet getRequest = new HttpGet(loginURL);
		getRequest.setConfig(noRedirectRequestConfig);
		try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
			loginURL = new URL(new URL(loginURL), httpResponse.getFirstHeader("Location").getValue()).toString();
			for (Header header : httpResponse.getHeaders("Set-Cookie")) {
				String cookie = header.getValue().split(";")[0];
				if (cookie.startsWith("prov=")) {
					providenceCookie = cookie;
					break;
				}
			}
		}
		if (providenceCookie == null) {
			throw new Exception("No providence cookie found.");
		}

		// Load login form
		getRequest = new HttpGet(loginURL);
		getRequest.setHeader("Cookie", providenceCookie);
		String fkey, loginActionURL;
		try (CloseableHttpResponse httpResponse = client.execute(getRequest);
				InputStream inputStream = httpResponse.getEntity().getContent()) {
			Document document = Jsoup.parse(inputStream, "UTF-8", loginURL);
			loginActionURL = document.getElementById("login-form").absUrl("action");
			fkey = document.selectFirst("input[name='fkey']").val();
		}

		// Submit login form
		postRequest = new HttpPost(loginActionURL);
		postRequest.setHeader("Cookie", providenceCookie);
		params = new ArrayList<NameValuePair>();
		params.add(new BasicNameValuePair("fkey", fkey));
		params.add(new BasicNameValuePair("email", EMAIL));
		params.add(new BasicNameValuePair("password", PASSWORD));
		postRequest.setEntity(new UrlEncodedFormEntity(params));
		String redirectURL, accountCookie = null;
		try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
			redirectURL = httpResponse.getFirstHeader("Location").getValue();
			for (Header header : httpResponse.getHeaders("Set-Cookie")) {
				String cookie = header.getValue().split(";")[0];
				if (cookie.startsWith("acct=")) {
					accountCookie = cookie;
					break;
				}
			}
		}
		if (accountCookie == null) {
			throw new Exception("No account cookie found.");
		}

		// OAuth2 redirection
		String location = redirectURL, authenticationCookie = null;
		// 1. https://stackoverflow.com/oauth?client_id=...
		// 2. https://data.stackexchange.com/user/oauth/stackapps?code=...
		// => /
		for (int i = 1; i <= 2; i++) {
			// the state parameter contains JSON, and Java doesn't like that unless it's %-encoded:
			URL url = new URL(URLDecoder.decode(location, "UTF-8"));
			URI uri = new URI(url.getProtocol(), url.getUserInfo(), url.getHost(), url.getPort(), url.getPath(), url.getQuery(), url.getRef());
			getRequest = new HttpGet(uri);
			getRequest.addHeader("Cookie", providenceCookie + (i == 1 ? "; " + accountCookie : ""));
			getRequest.setConfig(noRedirectRequestConfig);
			try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
				location = httpResponse.getFirstHeader("Location").getValue();
				System.out.println(location);
				if (i == 2) {
					for (Header header : httpResponse.getHeaders("Set-Cookie")) {
						String cookie = header.getValue().split(";")[0];
						if (cookie.startsWith(".ASPXAUTH=")) {
							authenticationCookie = cookie;
							break;
						}
					}
				}
			}
		}
		if (authenticationCookie == null) {
			throw new Exception("No authentication cookie found.");
		}

		// Execute query
		int siteID = 1; // 1 = Stack Overflow; see https://data.stackexchange.com/sites
		int queryID = 748298, revisionID = 1445524;
		postRequest = new HttpPost(
				"https://data.stackexchange.com/query/run/" + siteID + "/" + queryID + "/" + revisionID);
		params = new ArrayList<NameValuePair>();
		params.add(new BasicNameValuePair("badgename", "Legendary"));
		postRequest.setEntity(new UrlEncodedFormEntity(params));
		JSONObject jRoot;
		String jobID;
		try (CloseableHttpResponse httpResponse = client.execute(postRequest)) {
			jRoot = new JSONObject(IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8"));
			jobID = jRoot.has("job_id") ? jRoot.getString("job_id") : null;
			if (jobID != null) {
				// Results not in cache yet
				System.out.println("Starting job: " + jobID);
			}
		}
		while (jRoot.optBoolean("running")) {
			Thread.sleep(1000);
			// Poll job
			getRequest = new HttpGet(
					"https://data.stackexchange.com/query/job/" + jobID + "?_=" + new Date().getTime());
			try (CloseableHttpResponse httpResponse = client.execute(getRequest)) {
				jRoot = new JSONObject(IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8"));
			}
		}

		// Process results
		if (jRoot.has("resultSets")) {
			String messages = jRoot.getString("messages");
			int totalResults = jRoot.getInt("totalResults"); // NOTE: only populated when the results weren't cached
			int executionTime = jRoot.getInt("executionTime");
			JSONArray jResultSets = jRoot.getJSONArray("resultSets");
			for (int i = 0; i < jResultSets.length(); i++) {
				JSONObject jResultSet = jResultSets.getJSONObject(i);
				JSONArray jColumns = jResultSet.getJSONArray("columns");
				for (int j = 0; j < jColumns.length(); j++) {
					JSONObject jColumn = jColumns.getJSONObject(j);
					String name = jColumn.getString("name"), type = jColumn.getString("type");
				}
				JSONArray jRows = jResultSet.getJSONArray("rows");
				for (int j = 0; j < jRows.length(); j++) {
					JSONArray jRow = jRows.getJSONArray(j);
				}
			}
		} else {
			String error = jRoot.has("error") ? jRoot.getString("error") : null;
		}
		System.out.println(jRoot);

		System.exit(0);
	}

	private static final String EMAIL = "info@example.com", PASSWORD = "secret";
}