Github mirror script

github_mirror.py

#!/usr/bin/python
#
# Script to mirror github repositories and keep them up-to-date
#
# Usage:
#
# ./github_mirror
#
# It mirrors the configured repositories in the current directory
#
# (c) 2012 Dennis Kaarsemaker <dennis@kaarsemaker.net>

users = {
    'seveas': '*',
#    'puppetlabs': [
#        'puppet',
#        'facter',
#    ],
#    'BrightCoveOS': [
#        'Diamond',
#    ],
}

import urllib2
import json
import os
import subprocess
import sys
import whelk
import functools

git = functools.partial(whelk.shell.git, stderr=None)

def mirror(users):
    for user in users:
        mirror_user(user, users[user])

def mirror_user(user, repos):
    if not os.path.exists(user):
        os.mkdir(user)
    os.chdir(user)
    data = urllib2.urlopen('https://api.github.com/users/%s/repos' % user).read()
    all_repos = json.loads(data)
    for r in all_repos:
        if repos == '*' or r['name'] in repos:
            clone_repo(r)
    os.chdir('..')

def clone_repo(repo):
    if not os.path.exists(repo['name'] + '.git'):
        git('clone', '--bare', '-q', repo['git_url'])
        os.chdir(repo['name'] + '.git')
        git('config', 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*')
        os.chdir('..')
    else:
        os.chdir(repo['name'] + '.git')
        git('config', 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*')
        git('fetch', '-q', 'origin')
        git('remote', 'prune', 'origin')

        branches = git('branch', '--remote').stdout
        for branch in branches.split('\n'):
            branch = branch.strip().replace('origin/','')
            git('fetch','-q', 'origin','%s:%s' % (branch,branch))
        os.chdir('..')

    with open(os.path.join(repo['name'] + '.git', 'description'), 'w') as fd:
        fd.write(repo['description'].encode('utf-8'))
    with open(os.path.join(repo['name'] + '.git', 'cloneurl'), 'w') as fd:
        fd.write('%s<br />\n%s' % (repo['git_url'], repo['clone_url']))

mirror(users)