Clone an org's public repos to another org.

copier.py

import os
import json
import time
from datetime import datetime
from github import (
    Github, UnknownObjectException, GithubException, RateLimitExceededException
)

RATE_LIMIT_COUNT = 0


def rate_limit(func, *args, **kwargs):
    global RATE_LIMIT_COUNT
    try:
        return func(*args, **kwargs)
    except RateLimitExceededException:
        pass
    RATE_LIMIT_COUNT += 1
    print("limited: %s times" % RATE_LIMIT_COUNT)
    time.sleep(60)
    try:
        return func(*args, **kwargs)
    except RateLimitExceededException as e:
        print("Rate limit exceeded twice")
        raise e


def safe_fork(org, repo):
    try:
        return rate_limit(org.create_fork, repo)
    except GithubException as e:
        print("Skipping %s, error: %s" % (repo.name, e))
        return None


def safe_get_repo(org, repo_name):
    try:
        return rate_limit(org.get_repo, repo_name)
    except UnknownObjectException:
        print("Couldn't find repo %s" % repo_name)
        return None


def safe_get_branch_ref(repo, ref):
    try:
        return rate_limit(repo.get_git_ref, ref="heads/%s" % ref)
    except UnknownObjectException:
        return None


def main(token):
    g = Github(token)

    src_org = g.get_organization("alphagov")
    dst_org = g.get_organization("alphagov-mirror")

    new_repos = []
    seen_repos = []
    updated_repos = []

    for src_repo in src_org.get_repos('all'):
        repo_name = src_repo.name
        seen_repos.append(repo_name)

        print("\nSyncing %s..." % repo_name)

        dst_repo = safe_get_repo(dst_org, repo_name)

        if dst_repo is None:
            dst_repo = safe_fork(dst_org, src_repo)
            new_repos.append(repo_name)

        for src_branch in src_repo.get_branches():
            print(" - %s" % src_branch.name)

            dst_ref = safe_get_branch_ref(dst_repo, src_branch.name)

            if dst_ref:
                # Apparently sometimes branch commit or dst_ref.object is None
                if src_branch.commit and dst_ref.object and \
                   src_branch.commit.sha != dst_ref.object.sha:

                    rate_limit(dst_ref.edit, sha=src_branch.commit.sha,
                               force=True)
                    updated_repos.append(repo_name)

                elif src_branch.commit is None or dst_ref.object is None:
                    print("one is None: %s %s" %
                          (src_branch.commit, dst_ref.object))
            else:
                rate_limit(dst_repo.create_git_ref,
                           ref="refs/heads/%s" % src_branch.name,
                           sha=src_branch.commit.sha)
                updated_repos.append(repo_name)
            # Because I can't be bothered to make rate limiting work elegantly
            time.sleep(0.1)
        time.sleep(1)

    return {
        "datetime": datetime.utcnow(),
        "new_repos": new_repos,
        "updated_repos": updated_repos,
        "seen_repos": seen_repos
    }


if __name__ == '__main__':
    metadata = main(os.getenv("GITHUB_TOKEN"))

    print(json.dumps(metadata))