Last active
August 29, 2015 14:03
-
-
Save arienkock/58efbc6f0e1426546e41 to your computer and use it in GitHub Desktop.
A fixed rate limiter for HTTP requests. Can be used to avoid being blocked by throttle-limited API's like Facebook.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package net.webapp.util; | |
import java.io.IOException; | |
import java.net.URI; | |
import java.net.URISyntaxException; | |
import java.util.ArrayList; | |
import org.apache.http.HttpException; | |
import org.apache.http.HttpRequest; | |
import org.apache.http.HttpRequestInterceptor; | |
import org.apache.http.protocol.HttpContext; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import com.google.api.client.util.Lists; | |
public class FixedRateInterceptor implements HttpRequestInterceptor { | |
private static final Logger logger = LoggerFactory | |
.getLogger(FixedRateInterceptor.class); | |
private ArrayList<Long> samples = Lists.newArrayList(); | |
private String host; | |
private double maxRate; | |
public FixedRateInterceptor(double rate, String host) { | |
this.host = host; | |
this.maxRate = rate; | |
} | |
public synchronized void process(HttpRequest request, HttpContext context) | |
throws HttpException, IOException { | |
String requestHost; | |
try { | |
requestHost = new URI(context.getAttribute("http.target_host") | |
.toString()).getHost(); | |
} catch (URISyntaxException e1) { | |
throw new IOException(e1); | |
} | |
if (requestHost.equalsIgnoreCase(host)) { | |
while (rate() > maxRate) { | |
if (logger.isDebugEnabled()) { | |
logger.debug("rate {} maxRate {}", rate(), maxRate); | |
} | |
try { | |
Thread.sleep(500); | |
} catch (InterruptedException e) { | |
} | |
} | |
sample(); | |
} | |
} | |
private synchronized double rate() { | |
Long now = System.currentTimeMillis(); | |
if (samples.size() == 0 || now.equals(samples.get(0))) { | |
return 0; | |
} | |
if (logger.isDebugEnabled()) { | |
logger.debug("samples {} {} now {}", samples.size(), | |
samples.get(0), now); | |
} | |
return (samples.size() * 1000D) / (now - samples.get(0)); | |
} | |
private synchronized void sample() { | |
samples.add(System.currentTimeMillis()); | |
if (samples.size() > 20) { | |
samples.remove(0); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment