Script to mirror github repos, designed to be run as a cron -- usage: $ mirror-github.sh bob/pet_proj dave/ fred/cool_proj

mirror-github.sh

#!/bin/bash -eu

# Mirrors GitHub repos
# Takes a list of repos to mirror formatted as "user/reponame".
# The reponame can be omitted to mirror all repos for a user or organisation e.g. "youdevise/"
# The repos for a team can be mirrored using the syntax "team:orgname/teamname

#FIXME these might be better as arguments
MIRROR_DIR="${MIRROR_DIR:-/tmp/githubhmirror}"
GITHUB_USER="${GITHUB_USER:-}"
JENKINS_URL="${JENKINS_URL:-http://jenkins.example.com}"
MIRROR_URL="${MIRROR_URL:-http://git.example.com/git/github}"

LOCKFILE="/var/lock/mirror-github"
SUCCESSFILE="git-daemon-export-ok"

GITHUB_CLONE_URL="git@github.com:"
GITHUB_API_URL="https://api.github.com"

CURL_CMD="curl --silent --show-error"

# Authenticate so we get 5000 API requests per hour (instead of 60)
GITHUB_CURL_COMMAND="${CURL_CMD} ${GITHUB_USER:+--user ${GITHUB_USER}}"


github_api_call() {
    URL="${1}"
    JQ_COMMAND="${2}"

    HEADER_FILE="$( mktemp )"
    DATA_FILE="$( mktemp )"

    NEXT="${URL}?per_page=100"
    while [ "${NEXT}" ]; do
        ${GITHUB_CURL_COMMAND} --silent --dump-header "${HEADER_FILE}" "${NEXT}" > "${DATA_FILE}"
        NEXT="$( grep 'Link:' < "${HEADER_FILE}" | sed -nr 's/Link:.*<(.*?)>; rel="next".*/\1/p' )"
        jq -r "${JQ_COMMAND}" <"${DATA_FILE}"
    done

    rm "${HEADER_FILE}" "${DATA_FILE}"
}

fetch_repos_for_user() {
    github_api_call "${GITHUB_API_URL}/users/${1}/repos" '.[] | .name'
}

fetch_repos_for_team() {
    TEAM_ID="$(github_api_call "${GITHUB_API_URL}/orgs/${1}/teams" ".[] | select(.name==\"${2}\") | .id")"
    github_api_call "${GITHUB_API_URL}/teams/${TEAM_ID}/repos" '.[] | .name'
}

create_mirror() {
    echo "mirroring ${1}/${2}"
    REPO_DIR="${MIRROR_DIR}/${1}/${2}.git"
    timeout 1200 git clone --mirror --quiet "${GITHUB_CLONE_URL}${1}/${2}.git" "${REPO_DIR}"
    touch "${REPO_DIR}/${SUCCESSFILE}"
}

refresh_mirror() {
    echo "updating mirror of ${1}/${2}"
    REPO_DIR="${MIRROR_DIR}/${1}/${2}.git"
    BEFORE="$(git -C "${REPO_DIR}" for-each-ref --shell --format='%(refname)' | xargs git -C "${REPO_DIR}" rev-parse)"
    timeout 600 git -C "${REPO_DIR}" fetch --prune --quiet
    AFTER="$(git -C "${REPO_DIR}" for-each-ref --shell --format='%(refname)' | xargs git -C "${REPO_DIR}" rev-parse)"
    touch "${REPO_DIR}/${SUCCESSFILE}"

    if [ "${BEFORE}" != "${AFTER}" ]; then
        echo "detected changes... informing jenkins"
        ${CURL_CMD} "${JENKINS_URL}/git/notifyCommit?url=${MIRROR_URL}/${1}/${2}.git"
    fi
}

process_repo() {
    if [ -d "${MIRROR_DIR}/${1}/${2}.git" ]; then
        (refresh_mirror "${1}" "${2}")
    else
        (create_mirror "${1}" "${2}")
    fi
}

process_userrepoteam() {
    if [ -z "${1##*:*}" ]; then
      TYPE="${1%%:*}"
    elif [ -z "${1##*/*}" ]; then
      TYPE='repo'
    else
      TYPE='user'
    fi

    DATA="${1#*:}"
    USER_OR_ORG="${DATA%%/*}"
    QUALIFIER="${DATA#*/}"

    case "${TYPE}" in
      repo)
        process_repo "${USER_OR_ORG}" "${QUALIFIER}"
      ;;
      user)
        for REPO in $(fetch_repos_for_user "${USER_OR_ORG}"); do
          process_repo "${USER_OR_ORG}" "${REPO}"
        done
      ;;
      team)
        for REPO in $(fetch_repos_for_team "${USER_OR_ORG}" "${QUALIFIER}"); do
          process_repo "${USER_OR_ORG}" "${REPO}"
        done
      ;;
    esac
}

# Obtain lock and execute
(
    flock -xn 200
    mkdir -p "${MIRROR_DIR}"
    trap 'echo "ERROR: Failed to process ${LINE}" >&2' ERR
    for LINE in "$@"
    do
        process_userrepoteam "${LINE}"
    done
) 200>"${LOCKFILE}"