Gitlab: Clone / Pull all projects in a group

gitlab-sync-projects

#!/bin/sh
# Source
# https://gist.github.com/JonasGroeger/1b5155e461036b557d0fb4b3307e1e75

# API Documentation
# https://docs.gitlab.com/ce/api/projects.html#list-projects

GITLAB_URL="https://{GITLAB_URL}/"
GITLAB_PRIVATE_TOKEN=""
NAMESPACE="$1"
PROJECT_SEARCH_PARAM=""
REPO_FILENAME="repos.json"

PROJECT_PROJECTION="{ "path": .path_with_namespace, "git": .ssh_url_to_repo }"


confirm() {
    # call with a prompt string or use a default
    read "response?Are you sure? [y/N]"
    case "$response" in
        [yY][eE][sS]|[yY]) 
            true
            ;;
        *)
            false
            ;;
    esac
}

if [ -z "$NAMESPACE" ]; then
  echo "If you don't provide a namespace/filter parameter all projects will be synced."
  confirm || exit 1
  PROJECT_SELECTION="."
else
  PROJECT_SELECTION="select(.namespace.full_path|contains(\"$NAMESPACE\"))"
fi  

if [ -z "$GITLAB_PRIVATE_TOKEN" ]; then
    echo "Please set the variable GITLAB_PRIVATE_TOKEN in $0"
    echo "See ${GITLAB_URL}profile/personal_access_tokens"
    exit 1
fi

CHECK_HOST=$(curl -sf -o /dev/null $GITLAB_URL; echo $?)
if [ $CHECK_HOST != 0 ]; then
    echo "Error connecting to GITLAB_URL=$GITLAB_URL"
    exit 1
fi

trap "{ rm -f $REPO_FILENAME; }" EXIT

# clean up any old files since we'll now be appending to the file
[ -e $REPO_FILENAME  ] && rm $REPO_FILENAME

PAGE_COUNTER=1
while true; do
    echo "Reading page $PAGE_COUNTER"

    CURL_OUT=$(curl -s -k "${GITLAB_URL}api/v4/projects?archived=false&private_token=$GITLAB_PRIVATE_TOKEN&search=$PROJECT_SEARCH_PARAM&per_page=999&page=$PAGE_COUNTER")
    if [ "$CURL_OUT" = "[]" ]; then break; fi

    echo $CURL_OUT | tr -d '\n' | tr -d '\r' | jq --raw-output --compact-output ".[] | $PROJECT_SELECTION | $PROJECT_PROJECTION" >> "$REPO_FILENAME"
    let PAGE_COUNTER++
done

while read repo; do
    THEPATH=$(echo "$repo" | jq -r ".path")
    GIT=$(echo "$repo" | jq -r ".git")
    
    if [ ! -d "$THEPATH" ]; then
	mkdir -p $THEPATH
        echo "Cloning $THEPATH ( $GIT )"
        git clone "$GIT" "$THEPATH"
    else
        echo "Pulling $THEPATH"
        (cd "$THEPATH" && git pull)
    fi
done < "$REPO_FILENAME"

wait