Skip to content

Instantly share code, notes, and snippets.

@SamLR
Created Sep 1, 2019
Embed
What would you like to do?
Clone an org's public repos to another org.
import os
import json
import time
from datetime import datetime
from github import (
Github, UnknownObjectException, GithubException, RateLimitExceededException
)
RATE_LIMIT_COUNT = 0
def rate_limit(func, *args, **kwargs):
global RATE_LIMIT_COUNT
try:
return func(*args, **kwargs)
except RateLimitExceededException:
pass
RATE_LIMIT_COUNT += 1
print("limited: %s times" % RATE_LIMIT_COUNT)
time.sleep(60)
try:
return func(*args, **kwargs)
except RateLimitExceededException as e:
print("Rate limit exceeded twice")
raise e
def safe_fork(org, repo):
try:
return rate_limit(org.create_fork, repo)
except GithubException as e:
print("Skipping %s, error: %s" % (repo.name, e))
return None
def safe_get_repo(org, repo_name):
try:
return rate_limit(org.get_repo, repo_name)
except UnknownObjectException:
print("Couldn't find repo %s" % repo_name)
return None
def safe_get_branch_ref(repo, ref):
try:
return rate_limit(repo.get_git_ref, ref="heads/%s" % ref)
except UnknownObjectException:
return None
def main(token):
g = Github(token)
src_org = g.get_organization("alphagov")
dst_org = g.get_organization("alphagov-mirror")
new_repos = []
seen_repos = []
updated_repos = []
for src_repo in src_org.get_repos('all'):
repo_name = src_repo.name
seen_repos.append(repo_name)
print("\nSyncing %s..." % repo_name)
dst_repo = safe_get_repo(dst_org, repo_name)
if dst_repo is None:
dst_repo = safe_fork(dst_org, src_repo)
new_repos.append(repo_name)
for src_branch in src_repo.get_branches():
print(" - %s" % src_branch.name)
dst_ref = safe_get_branch_ref(dst_repo, src_branch.name)
if dst_ref:
# Apparently sometimes branch commit or dst_ref.object is None
if src_branch.commit and dst_ref.object and \
src_branch.commit.sha != dst_ref.object.sha:
rate_limit(dst_ref.edit, sha=src_branch.commit.sha,
force=True)
updated_repos.append(repo_name)
elif src_branch.commit is None or dst_ref.object is None:
print("one is None: %s %s" %
(src_branch.commit, dst_ref.object))
else:
rate_limit(dst_repo.create_git_ref,
ref="refs/heads/%s" % src_branch.name,
sha=src_branch.commit.sha)
updated_repos.append(repo_name)
# Because I can't be bothered to make rate limiting work elegantly
time.sleep(0.1)
time.sleep(1)
return {
"datetime": datetime.utcnow(),
"new_repos": new_repos,
"updated_repos": updated_repos,
"seen_repos": seen_repos
}
if __name__ == '__main__':
metadata = main(os.getenv("GITHUB_TOKEN"))
print(json.dumps(metadata))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment