Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Mirror all my github repositories and gists
#!/usr/bin/env python3
# See
# You can find the latest version of this script at
import os
import sys
import json
import urllib.request
import subprocess
__version__ = '0.4'
__author__ = 'Marius Gedminas <>'
__url__ = ''
# configuration
gists_of = ['mgedmin']
repos_of = ['mgedmin', 'gtimelog']
backup_dir = os.path.expanduser('~/github')
gist_backup_dir = os.path.expanduser('~/github/gists')
# helpers
class Error(Exception):
"""An error that is not a bug in this script."""
def ensure_dir(dir):
if not os.path.isdir(dir):
def get_json_and_headers(url):
"""Perform HTTP GET for a URL, return deserialized JSON and headers.
Returns a tuple (json_data, headers) where headers is an instance
of email.message.Message (because that's what urllib gives us).
with urllib.request.urlopen(url) as r:
# We expect Github to return UTF-8, but let's verify that.
content_type ='Content-Type', '').lower()
if content_type not in ('application/json; charset="utf-8"',
'application/json; charset=utf-8'):
raise Error('Did not get UTF-8 JSON data from {0}, got {1}'
.format(url, content_type))
return json.loads('UTF-8')),
def get_github_list(url, batch_size=100):
"""Perform (a series of) HTTP GETs for a URL, return deserialized JSON.
Format of the JSON is documented at
Supports batching (which Github indicates by the presence of a Link header,
e.g. ::
Link: <>; rel="next",
<>; rel="last"
# API documented at
res, headers = get_json_and_headers('{0}?per_page={1}'.format(
url, batch_size))
page = 1
while 'rel="next"' in headers.get('Link', ''):
page += 1
more, headers = get_json_and_headers('{0}?page={1}&per_page={2}'.format(
url, page, batch_size))
res += more
return res
def info(*args):
print(" ".join(map(str, args)))
def backup(git_url, dir):
if os.path.exists(dir):['git', 'fetch'], cwd=dir)
else:['git', 'clone', '--mirror', git_url])
def update_description(git_dir, description):
with open(os.path.join(git_dir, 'description'), 'w', encoding='UTF-8') as f:
f.write(description + '\n')
def update_cloneurl(git_dir, cloneurl):
with open(os.path.join(git_dir, 'cloneurl'), 'w') as f:
f.write(cloneurl + '\n')
def back_up_gists_of(username, gist_backup_dir=gist_backup_dir):
for gist in get_github_list('' % username):
dir = gist['id'] + '.git'
description = gist['description'] or "(no description)"
info("+", "gists/" + gist['id'], "-", description.partition('\n')[0])
backup(gist['git_pull_url'], dir)
update_description(dir, description + '\n\n' + gist['html_url'])
update_cloneurl(dir, gist['git_push_url'])
def back_up_repos_of(username, backup_dir=backup_dir):
for repo in get_github_list('' % username):
dir = repo['name'] + '.git'
description = repo['description'] or "(no description)"
info("+", repo['full_name'])
backup(repo['git_url'], dir)
update_description(dir, description + '\n\n' + repo['html_url'])
update_cloneurl(dir, repo['ssh_url'])
# action
if __name__ == '__main__':
for user in gists_of:
for user in repos_of:

mgedmin commented Feb 25, 2013

Update: use git clone --mirror, so we get bare repositories and direct refs for all the branches.


mgedmin commented Mar 5, 2013

Update: mirror gists as well.

Thanks, this is really handy, have created a minor fork if you wanted to merge the changes in - simply allows the exclusion of specific repos -


mgedmin commented Sep 29, 2014

Update: backs up repos of multiple GitHub users/organizations.


mgedmin commented Nov 23, 2014

Version 0.4: support more than 100 GitHub repositories (i.e. implement API pagination).

How to run it with private repos? There is no var for password.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment