gitlab-clone-or-pull-all-repos.sh

gistfile1.txt

#!/bin/bash

GITLAB_TOKEN=<gitlab token>

function get_repos(){
    page=$1
    repos=($(curl --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "https://gitlab.com/api/v4/projects/?owned=true&per_page=100&page=${page}" | jq -r '.[].ssh_url_to_repo')) 
    echo "${repos[@]}"
}

# Initialize a semaphore with a given number of tokens
open_sem(){
    mkfifo pipe-$$
    exec 3<>pipe-$$
    rm pipe-$$
    local i=$1
    for((;i>0;i--)); do
        printf %s 000 >&3
    done
}

# Run the given command asynchronously and pop/push tokens
run_with_lock(){
    local x
    # this read waits until there is something to read
    read -u 3 -n 3 x && ((0==x)) || exit $x
    (
     ( "$@"; )
    # push the return code of the command to the semaphore
    printf '%.3d' $? >&3
    )&
}

# Repo clone or fetch
function clone_or_fetch(){
	repo_url=$1
    repo=$(echo "$repo_url" | awk -F'/' '{print $NF}' | sed 's/\.git//')

	if [ -d $repo ]; then # repo exists
 		((cd $repo && git fetch --no-tags --prune origin && git stash save && git checkout dev && git pull origin dev && cd .. ) | true)
	else
		git clone $repo_url
	fi
}


i=1
page=1
N=4
open_sem $N
while [[ true ]]; do
    repos="$(get_repos $page)"
    if [ -z "$repos" ]; then
        break;
    fi
    for repo in ${repos[@]}; do
        echo $i: $repo
        run_with_lock   clone_or_fetch $repo
        i=$(($i+1))
    done
    page=$(($page+1))
done