Dump all git repositories

git-dump.py

## Dump all git repositories from selected users. In this example notabug.org and github.com are used
## Change the dictionary "repos" to your own data
## Needs https://gitpython.readthedocs.io/
## pip install GitPython

import git, logging, os
from git import Git, Repo, Remote, RemoteProgress

repos = {
    'notabug': {
        'url': 'notabug.org',
        'desci': [
            'tercasemfim',
            'scripts',
            'doeumaaula',
            'amplifico',
            'cademeuposto',
        ],
        'matehackers': [
            'cademeuposto',
        ],
    },
    'github': {
        'url': 'github.com',
        'iuriguilherme': [
            'bin',
            'tercasemfim',
            'tg-cryptoforexbot',
            'doeumaaula',
        ],
        'MetaReciclagem': [
            'MetaReciclagem.github.io',
        ],
        'matehackers': [
            'matebot',
            'mateonline',
            'denise',
        ],
    },
}

logging.basicConfig(level=logging.INFO)

## Current directory
cwd = os.getcwd()

## This is ugly but works
class Progress(RemoteProgress):
    def update(self, op_code, cur_count, max_count=None, message=''):
        print(
            '{}: '.format(op_code),
            '{}%'.format((cur_count / (max_count or 100.0)* 100)),
            message or "",
        )

for site in repos:
    logging.info("Cloning from site {}...".format(repos[site]['url']))
    for user in (user for user in repos[site] if user != 'url'):
        try:
            logging.info("Cloning from organization {}...".format(user))
            os.mkdir(os.path.join(user))
        except FileExistsError:
            pass
        except Exception as e:
            logging.exception(repr(e))
        for repo in repos[site][user]:
            try:
                logging.info("Cloning repository {}...".format(repo))
                os.mkdir(os.path.join(user, repo))
            except FileExistsError:
                pass
            except Exception as e:
                logging.exception(repr(e))
            try:
                git_repo = Repo.init(os.path.join(cwd, user, repo))
                with git_repo.git.custom_environment(
                    GIT_SSH_COMMAND="ssh -i {}".format(
                    os.path.join(os.path.expanduser("~"), '.ssh', 'id_rsa'))
                ):
                    git_remote = None
                    ## if the remote doesn't exist, we create it
                    try:
                        git_remote = git_repo.remote(site)
                    except:
                        git_remote = git_repo.create_remote(
                            site, "git@{url}:{user}/{repo}.git".format(
                            url = repos[site]['url'], user = user, repo = repo)
                        )
                    logging.info("Branches: {}".format(' '.join([ref.name for ref in git_remote.refs])))
                    ## git fetch
                    for fetch_info in git_remote.fetch(progress=Progress()):
                        print("Updating {} to {}".format(fetch_info.ref, fetch_info.commit))
                    ## git pull using first remote branch
                    ## not assuming it's 'master'
                    ## but assuming the first one in the list is the correct one
                    try:
                        for pull_info in git_remote.pull(
                            [str(ref).rsplit('/')[-1] for ref in git_remote.refs][0],
                            progress = Progress(),
                        ):
                            print("{}".format(pull_info))
                    except IndexError:
                        logging.warning("Empty repository!")
            except Exception as e:
                logging.exception(repr(e))
                raise