Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
GET/POST HTTP request and HTML parsing with Jsoup library
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
public class SimpleTests {
public static void main(String[] args) {
try {
// for avoiding handshake alert: unrecognized_name
System.setProperty("jsse.enableSNIExtension", "false");
// WARNING: do it only if security isn't important, otherwise you have
// to follow this advices:
// Create a trust manager that does not validate certificate chains
TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager(){
public X509Certificate[] getAcceptedIssuers(){return null;}
public void checkClientTrusted(X509Certificate[] certs, String authType){}
public void checkServerTrusted(X509Certificate[] certs, String authType){}
// Install the all-trusting trust manager
try {
SSLContext sc = SSLContext.getInstance("TLS");
sc.init(null, trustAllCerts, new SecureRandom());
} catch (Exception e) {
// Authentication example
// 1. GET request for login form, hidden fields and cookies
Connection.Response loginForm = Jsoup.connect(MY_COMPANY_LOGIN_URL)
Document doc = loginForm.parse();
// eventually get hidden fields generated dinamically by the server
Elements urlFwdEl = doc.getElementsByAttributeValue("name", "urlFwd");
String forwardingUrl = urlFwdEl.get(0).val();
Elements tokenEl = doc.getElementsByAttributeValue("name", "tokenLogin");
String token = tokenEl.get(0).val();
// 2. POST authentication
Connection.Response auth = Jsoup.connect(MY_COMPANY_AUTH_URL)
.userAgent("Firefox ...")
.data("username", USERNAME)
.data("password", PASSWORD)
.data("urlFwd", forwardingUrl)
.data("tokenLogin", token)
.cookies(loginForm.cookies()) // important!
Document document = auth.parse();
String htmlAuth = document.toString();
Document docPage = Jsoup.parse(htmlAuth);
// repeat this methods combination
} catch (IOException e) {
// TODO Auto-generated catch block
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.