Last active
July 9, 2018 16:14
-
-
Save splattael/e70ef44322ca5732015f1c3a0ec0f62d to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/ruby | |
gem "http" | |
require "http" | |
require "json" | |
ARGV.size == 3 or abort "usage: #$0 username github_token start-date" | |
USERNAME, TOKEN, START_DATE = *ARGV | |
HEADERS = { | |
"Authorization" => "token #{TOKEN}", | |
"Accept" => "application/vnd.github.cloak-preview" | |
} | |
QUERY = "https://api.github.com/search/commits?q=author:splattael+author-date:%{from}..%{to}" | |
def rate_limit(response) | |
limit = response.headers["X-RateLimit-Limit"].to_i | |
remaining = response.headers["X-RateLimit-Remaining"].to_i | |
reset_at = Time.at(response.headers["X-RateLimit-Reset"].to_i) | |
retry_after = response.headers["Retry-After"].to_i | |
diff = (reset_at - Time.now).abs | |
sleep = remaining == 0 ? -1 : diff / remaining | |
$stdout.puts "#{remaining}/#{limit} -> #{reset_at} -> (#{diff}s) #{sleep}s" | |
return sleep, retry_after | |
end | |
def paginate(response) | |
if response.headers["Link"] =~ /<([^>]+?)>; rel="next"/ | |
$1 | |
end | |
end | |
def with_rate_limit | |
response = yield | |
sleep, retry_after = rate_limit(response) | |
while retry_after > 0 | |
$stdout.puts "Retry after: #{retry_after}s... cheating" | |
Kernel.sleep 5 | |
response = yield | |
sleep, retry_after = rate_limit(response) | |
end | |
while JSON(response).fetch("incomplete_results") | |
$stdout.puts "INCOMPLETE retry" | |
Kernel.sleep(sleep) | |
response = yield | |
sleep, retry_after = rate_limit(response) | |
end | |
Kernel.sleep(sleep) | |
response | |
end | |
def fetch_for(from, to) | |
query = QUERY % { | |
from: from.strftime, | |
to: to.strftime, | |
} | |
year = from.year | |
file = "commits-#{year}.json" | |
File.open(file, "w") do |fh| | |
while query | |
response = with_rate_limit do | |
$stdout.puts query.inspect | |
HTTP.get(query, headers: HEADERS) | |
end | |
fh.puts(response.body) | |
query = paginate(response) | |
end | |
end | |
end | |
date = Date.parse(START_DATE) | |
while date < Date.today | |
next_week = date + 7 | |
fetch_for(date, next_week) | |
date = next_week | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment