Backup public github repositories

github_backup.py

#!/usr/bin/python

import sys
import os
import os.path
import re
from subprocess import check_call

# pip install PyGithub
from github import Github

REAL_FILE = os.path.abspath(__file__)
DIR = os.path.dirname(REAL_FILE)
SCRIPT_NAME = os.path.basename(REAL_FILE)
LOG_NAME = "log"
VENV_NAME = "venv"
REPO_RE = re.compile("^[a-zA-Z0-9\\-_]+$")
GIST_RE = re.compile("^[0-9a-zA-Z]+$")

# Send stdout from python to the log, and git from stdout,err to the log
# stderr from python can be mailed by cron.
log = open(os.path.join(DIR, LOG_NAME), "w", 1)
os.dup2(log.fileno(), sys.stdout.fileno())

user = Github().get_user("danielrichman")
repos = {}

for r in user.get_repos():
    name = r.name
    if REPO_RE.match(name) is None:
        raise ValueError("Bad repo name: " + name)
    name += ".git"
    clone = r.clone_url
    if name in repos:
        raise ValueError("Duplicate repo: " + name)
    repos[name] = clone

for r in user.get_gists():
    name = r.id
    if GIST_RE.match(name) is None:
        raise ValueError("Bad gist name: " + name)
    name += ".gist.git"
    clone = r.git_pull_url
    if name in repos:
        raise ValueError("Duplicate repo: " + name)
    repos[name] = clone

if SCRIPT_NAME in repos:
    raise ValueError("Can't overwrite myself")
if LOG_NAME in repos:
    raise ValueError("Can't overwrite the log")
if VENV_NAME in repos:
    raise ValueError("Can't overwrite the venv")

dirs = set(os.listdir(DIR))
dirs.remove(SCRIPT_NAME)
dirs.remove(LOG_NAME)
dirs.remove(VENV_NAME)

new = set(repos) - set(dirs)
update = set(repos) & set(dirs)
removed = set(dirs) - set(repos)

for name in new:
    print "Cloning", name, "from", repos[name]
    check_call(("git", "clone", "--mirror", repos[name], name),
               cwd=DIR, stdout=log, stderr=log)

for name in update:
    print "Updating", name
    check_call(("git", "fetch", "origin" ), cwd=os.path.join(DIR, name),
               stdout=log, stderr=log)
    check_call(("git", "gc"), cwd=os.path.join(DIR, name),
               stdout=log, stderr=log)

print "New repos:", ', '.join(new)
print "Updated repos:", ', '.join(update)
print "Removed repos!: ", ', '.join(removed)