Instantly share code, notes, and snippets.

Embed
What would you like to do?
package me.edrone.common.helpers;
import java.io.IOException;
public interface IWebClient {
public byte[] get(String url) throws IOException;
public byte[] head(String url) throws IOException;
public boolean isUrlAllowed(String url);
public boolean isPublicIpAddress(String url);
public boolean isPrivateIpAddress(String url);
}
package me.edrone.common.helpers;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
import com.amazonaws.services.ec2.model.Filter;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Reservation;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Random;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SquidProxyWebClientDecorator extends WebClientDecorator {
private static final Logger logger = LoggerFactory.getLogger(SquidProxyWebClientDecorator.class);
private final HttpClient httpClient2;
private final HttpClient httpClient1;
private final int PROXY_PORT = 3232;
private final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11";
private final List<String> ips = new ArrayList<>();
private final Random random = new Random();
public SquidProxyWebClientDecorator(IWebClient client) {
super(client);
RequestConfig params = RequestConfig
.custom()
.setRedirectsEnabled(true)
.setRelativeRedirectsAllowed(true)
.setConnectTimeout(30000)
.setSocketTimeout(30000)
.build();
ensureIps();
ListIterator<String> ipsIterator = ips.listIterator();
httpClient1 = HttpClients.custom()
.setDefaultRequestConfig(params)
.setUserAgent(USER_AGENT)
.setProxy(new HttpHost(ipsIterator.next(), PROXY_PORT))
.build();
httpClient2 = HttpClients.custom()
.setDefaultRequestConfig(params)
.setUserAgent(USER_AGENT)
.setProxy(new HttpHost(ipsIterator.next(), PROXY_PORT))
.build();
}
/**
*
* @param url to download
* @param httppost
* @return bytes[] if success or null otherwise
* @throws IOException
*/
private byte[] request(HttpGet httpget) throws IOException {
httpget.setHeader("Content-type", "application/x-www-form-urlencoded");
HttpEntity entity = null;
byte[] rawResponse = null;
try {
HttpResponse response = getRandomHttpClient().execute(httpget);
int statusCode = response.getStatusLine().getStatusCode();
entity = response.getEntity();
if (statusCode == 200) {
rawResponse = EntityUtils.toByteArray(entity);
}
} finally {
EntityUtils.consume(entity);
}
return rawResponse;
}
public HttpClient getRandomHttpClient() {
return random.nextInt(2) == 0 ? httpClient1 : httpClient2;
}
@Override
public byte[] get(String url) throws IOException {
HttpGet httpget = new HttpGet(url);
return request(httpget);
}
@Override
public byte[] head(String url) throws IOException {
HttpGet httpget = new HttpGet(url);
return request(httpget);
}
public java.net.URLConnection getProxyConnection(String url) throws IOException {
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(ips.get(random.nextInt(ips.size())), PROXY_PORT));
java.net.URLConnection c = new URL(url).openConnection(proxy);
c.setRequestProperty("User-Agent", USER_AGENT);
return c;
}
@Override
public boolean isUrlAllowed(String url) {
return iWebClient.isUrlAllowed(url);
}
@Override
public boolean isPublicIpAddress(String url) {
return iWebClient.isPublicIpAddress(url);
}
@Override
public boolean isPrivateIpAddress(String url) {
return iWebClient.isPrivateIpAddress(url);
}
private void ensureIps() {
AmazonEC2 ec2 = AmazonEC2ClientBuilder.standard().withRegion(Regions.EU_WEST_1).withCredentials(new DefaultAWSCredentialsProviderChain()).build();
DescribeInstancesResult result = ec2.describeInstances(
new DescribeInstancesRequest().withFilters(
new Filter().withName("tag:Name").withValues("FORWARD_PROXY"),
new Filter().withName("instance-state-name").withValues("running")
)
);
List<Reservation> reservations = result.getReservations();
for (Reservation reservation : reservations) {
List<Instance> instances = reservation.getInstances();
for (Instance instance : instances) {
ips.add(instance.getPrivateIpAddress());
}
}
}
}
package me.edrone.common.helpers;
public abstract class WebClientDecorator implements IWebClient{
protected IWebClient iWebClient;
public WebClientDecorator(IWebClient iWebClient) {
this.iWebClient = iWebClient;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment