clone or pull all repos for an organization on github

github-backup.py

#!/usr/bin/env python
import argparse, ConfigParser
import os, sys, subprocess
import yaml

from pygithub3 import Github
from datetime import datetime, date, time

import smtplib
from email.MIMEText import MIMEText

parser = argparse.ArgumentParser(description='backup github repositories')

parser.add_argument('-c', '--config', default="github.cfg", help="configuration file")

args = parser.parse_args()

config = ConfigParser.SafeConfigParser()
config.read(args.config)

# record the start time
time_start = datetime.now()

# login to github
gh = Github(login=config.get('github', 'username'), password=config.get('github', 'password'))

# get all the repositories for the specified organization
repos = gh.repos.list_by_org(config.get('github', 'org')).all()

results = {
  'duration': None,
  'repos': {}
}

for repo in repos:
  # build the paths that will be used
  base_path = os.path.abspath(os.path.join(config.get('system', 'path'), repo.owner.login))
  repo_path = os.path.join(base_path, repo.name)

  # set a repository name that accounts for forks
  repo_name = "%s/%s" % (repo.owner.login, repo.name)

  results['repos'][repo_name] = {}
  # create the base folders if needed
  if (not os.path.exists(base_path)):
    os.makedirs(base_path)

  try:
    p = None
    # check if the repository has already been cloned
    if (os.path.exists(repo_path)):
      # already have that repository, so do a pull
      cmd = "%s %s" % (config.get('system', 'cmd'), 'pull')
      p = subprocess.Popen(cmd, shell=True, cwd=repo_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    else:
      # "new" repository, so do a clone
      cmd = "%s %s %s %s" % (config.get('system', 'cmd'), 'clone', repo.ssh_url, repo.name)
      p = subprocess.Popen(cmd, shell=True, cwd=base_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # block for the output, should be multi-threaded 
    out, err = p.communicate()
    
    # add the output to results
    results['repos'][repo_name]['path'] = repo_path

    if (len(out.strip()) > 0):
      results['repos'][repo_name]['out'] = out.strip().split('\n')

    if (len(err.strip()) > 0):
      results['repos'][repo_name]['err'] = err.strip().split('\n')

  except subprocess.CalledProcessError, e:
    # add an error for the exception
    results['repos'][repo_name]['path'] = repo_path
    results['repos'][repo_name]['err'] = e.output
  except Exception, err:
    results['repos'][repo_name]['path'] = repo_path
    results['repos'][repo_name]['err'] = 'Unexpected exception: %s' % (str(err))

# for when we are all done!
results['duration'] = str(datetime.now() - time_start)

# build an email from the results object (using yaml to be lazy)
msg = MIMEText(yaml.safe_dump(results, indent=2, default_flow_style=False))
msg['subject'] = config.get('mail', 'subject')
msg['From'] = config.get('mail', 'from')
msg['To'] = config.get('mail', 'to')

smtp = smtplib.SMTP()

# connect to the smtp server
smtp.connect(config.get('mail', 'host'), config.get('mail', 'port'))

# check if username is set for sending
if (config.get('mail', 'username')):
  smtp.login(config.get('mail', 'username'), config.get('mail', 'password'))

smtp.sendmail(config.get('mail', 'from'), [config.get('mail', 'to')], msg.as_string())
smtp.quit()

github.cfg

[system]
cmd=/usr/bin/git
path=repos

[mail]
host=localhost
port=25
username=
password=
from=gitbackup@example.com
to=reports@example.com
subject=[GITBACKUP] Report

[github]
username=my-github-name
password=my-super-secret-password
org=example