all_commits.rb

#!/usr/bin/ruby

gem "http"
require "http"
require "json"

ARGV.size == 3 or abort "usage: #$0 username github_token start-date"

USERNAME, TOKEN, START_DATE = *ARGV

HEADERS = {
  "Authorization" => "token #{TOKEN}",
  "Accept" => "application/vnd.github.cloak-preview"
}
QUERY = "https://api.github.com/search/commits?q=author:splattael+author-date:%{from}..%{to}"

def rate_limit(response)
  limit = response.headers["X-RateLimit-Limit"].to_i
  remaining = response.headers["X-RateLimit-Remaining"].to_i
  reset_at = Time.at(response.headers["X-RateLimit-Reset"].to_i)
  retry_after = response.headers["Retry-After"].to_i

  diff = (reset_at - Time.now).abs
  sleep = remaining == 0 ? -1 : diff / remaining

  $stdout.puts "#{remaining}/#{limit} -> #{reset_at} -> (#{diff}s) #{sleep}s"

  return sleep, retry_after
end

def paginate(response)
  if response.headers["Link"] =~ /<([^>]+?)>; rel="next"/
    $1
  end
end

def with_rate_limit
  response = yield

  sleep, retry_after = rate_limit(response)

  while retry_after > 0
    $stdout.puts "Retry after: #{retry_after}s... cheating"
    Kernel.sleep 5
    response = yield
    sleep, retry_after = rate_limit(response)
  end

  while JSON(response).fetch("incomplete_results")
    $stdout.puts "INCOMPLETE retry"
    Kernel.sleep(sleep)
    response = yield
    sleep, retry_after = rate_limit(response)
  end

  Kernel.sleep(sleep)

  response
end

def fetch_for(from, to)
  query = QUERY % {
    from: from.strftime,
    to: to.strftime,
  }
  year = from.year
  file = "commits-#{year}.json"

  File.open(file, "w") do |fh|
    while query
      response = with_rate_limit do
        $stdout.puts query.inspect
        HTTP.get(query, headers: HEADERS)
      end

      fh.puts(response.body)

      query = paginate(response)
    end
  end
end

date = Date.parse(START_DATE)

while date < Date.today
  next_week = date + 7
  fetch_for(date, next_week)
  date = next_week
end