Calculate size of GitHub Archive data for a date range

ghsize.py

"""
Find out total size of the JSON files of GitHub Archive
in a particular date range.
"""

import requests

from datetime import timedelta, datetime

URL = "http://data.githubarchive.org/%s.json.gz"


def hour_range(start, end):
    for n in range(int((end - start).days)):
        for h in range(0, 24):
            yield start + timedelta(days=n, hours=h)


def human_bytes(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

start = datetime(2017, 1, 1)
end = datetime(2017, 10, 1)

total = 0

for hour in hour_range(start, end):

    url = URL % hour.strftime("%Y-%m-%d-%-H")

    size = int(requests.head(url).headers["Content-Length"])
    total += size

    print("%s,%s" % (human_bytes(size), url))

print(human_bytes(total))

python ghsize.py > GHA_2017.txt