Skip to content

Instantly share code, notes, and snippets.

@arienkock
Last active August 29, 2015 14:03
Show Gist options
  • Save arienkock/58efbc6f0e1426546e41 to your computer and use it in GitHub Desktop.
Save arienkock/58efbc6f0e1426546e41 to your computer and use it in GitHub Desktop.
A fixed rate limiter for HTTP requests. Can be used to avoid being blocked by throttle-limited API's like Facebook.
package net.webapp.util;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.protocol.HttpContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.api.client.util.Lists;
public class FixedRateInterceptor implements HttpRequestInterceptor {
private static final Logger logger = LoggerFactory
.getLogger(FixedRateInterceptor.class);
private ArrayList<Long> samples = Lists.newArrayList();
private String host;
private double maxRate;
public FixedRateInterceptor(double rate, String host) {
this.host = host;
this.maxRate = rate;
}
public synchronized void process(HttpRequest request, HttpContext context)
throws HttpException, IOException {
String requestHost;
try {
requestHost = new URI(context.getAttribute("http.target_host")
.toString()).getHost();
} catch (URISyntaxException e1) {
throw new IOException(e1);
}
if (requestHost.equalsIgnoreCase(host)) {
while (rate() > maxRate) {
if (logger.isDebugEnabled()) {
logger.debug("rate {} maxRate {}", rate(), maxRate);
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
}
}
sample();
}
}
private synchronized double rate() {
Long now = System.currentTimeMillis();
if (samples.size() == 0 || now.equals(samples.get(0))) {
return 0;
}
if (logger.isDebugEnabled()) {
logger.debug("samples {} {} now {}", samples.size(),
samples.get(0), now);
}
return (samples.size() * 1000D) / (now - samples.get(0));
}
private synchronized void sample() {
samples.add(System.currentTimeMillis());
if (samples.size() > 20) {
samples.remove(0);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment