Clone all projects from a GitLab group, even for +100 pages.

git_clone_all.sh

#!/usr/bin/env bash

set -o errexit
set -o pipefail
set -o nounset

#---------------------------------------------------------------------------------------------------------------------------
# INFO
#---------------------------------------------------------------------------------------------------------------------------
# ./git_clone_all.sh <group> <token> <url> <branch>

#---------------------------------------------------------------------------------------------------------------------------
# VARIABLES
#---------------------------------------------------------------------------------------------------------------------------
: "${DEBUG:=false}"
: "${IFS=$'\n\t'}"
: "${GROUP=${1:-}}"
: "${TOKEN=${2:-}}"
: "${GIT_URL=${3:-}}"
: "${BRANCH=${4:-}}"
: "${PREFIX:=ssh_url_to_repo}"
: "${GROUP_ID:=$(curl -sSL -H "PRIVATE-TOKEN: ${TOKEN}" "${GIT_URL}/api/v4/groups?search=${GROUP}" | jq '.[].id')}"
: "${PAGE_COUNTER:=1}"
: "${TOTAL_PAGES:=$(curl -Ssl --head -H "PRIVATE-TOKEN: ${TOKEN}" -X GET "${GIT_URL}/api/v4/groups/${GROUP_ID}/projects" | grep -i x-total-pages | sed -e 's/x-total-pages..\([0-9]*\).*/\1/g')}"

#---------------------------------------------------------------------------------------------------------------------------
# FUNCTIONS
#---------------------------------------------------------------------------------------------------------------------------
function _info ()  { >&2 printf '[ \033[00;34mINFO\033[0m ] %s\n' "$*"; }
function _warn ()  { >&2 printf '\033[2K[ \033[0;33mWARN\033[0m ] %s\n' "$*"; }
function _error () { >&2 printf '\033[2K[ \033[0;31mFAIL\033[0m ] %s\n' "$*"; }
function _debug () { >&2 printf '[ \033[00;37mDBUG\033[0m ] %s\n' "$*"; }

function _pre_checks {
  [[ -z "${GROUP}" ]]   && _error "a group name arg is required" && exit 1
  [[ -z "${GIT_URL}" ]] && _error "a gitlab URL is required." && exit 1
  [[ -z "${TOKEN}" ]]   && _error "an auth token arg is required. See ${GIT_URL}/profile/account" && exit 1
  [[ -z "${BRANCH}" ]]  && _error "a branch is required." && exit 1

  _commands_check jq git curl

  if [[ -d "${GROUP}/${BRANCH}" ]]; then
    read -p "Continuing will overwrite previous data. Are you sure? " -n 1 -r
    echo
    if [[ "${REPLY}" =~ ^[Nn]$ ]]; then
      exit
    elif [[ "${REPLY}" =~ ^[Yy]$ ]]; then
      rm -rf "${GROUP:?}/${BRANCH:?}"
      true
    else
      _error "Invalid input received (${REPLY})."
    fi
  fi
}

function _commands_check {
  : "${result:=}"
  for i; do
    if command -v "${i}" >/dev/null 2>&1; then
      _info "${i} is installed on the system. Continuing..."
      true
    else
      _error "This script requires '$i'"
      result=false
    fi
  done
  if [[ "${result}" == "false" ]]; then exit 1; else true; fi
}

function _get_projects {
  _info "Cloning all git projects in group ${GROUP} with group ID: ${GROUP_ID}";
  while [ "${PAGE_COUNTER}" -le "${TOTAL_PAGES}" ] ; do
    CONTENTS+=$(curl -sSL -H "PRIVATE-TOKEN: ${TOKEN}" -X GET "${GIT_URL}/api/v4/groups/${GROUP_ID}/projects?page=${PAGE_COUNTER}&per_page=100&archived=false" | jq --arg p "${PREFIX}" '.[] | .[$p]')
    PAGE_COUNTER=$((PAGE_COUNTER+1))
  done
  # old style, better, but ending in an endless loop, need to fix this later.
  #for ((i=1; ; i++)); do
  #  if CONTENTS+=$(curl -sSL -H "PRIVATE-TOKEN: ${TOKEN}" -X GET "${GIT_URL}/api/v4/groups/${GROUP_ID}/projects?page=${i}&per_page=100&archived=false" | jq --arg p "${PREFIX}" '.[] | .[$p]'); then
  #    echo "${i}"
  #  fi
  #done
}

function _clone_projects {
  mkdir -p "${GROUP}/${BRANCH}"
  pushd "${GROUP}/${BRANCH}" >/dev/null 2>&1
  for ITEM in "${CONTENTS[@]}"; do
    echo "${ITEM}" | jq . | xargs -L1 git clone -b "${BRANCH}"
  done
  popd >/dev/null 2>&1
}

# TEST FUNCTIONS
# function _update_existing_group {
#   if [[ -d "${GROUP}/${BRANCH}" ]]; then
#     pushd "${GROUP}/${BRANCH}" >/dev/null 2>&1
#     for d in */ ; do
#       _info "Updating: ${d}"
#       pushd "${d}" >/dev/null 2>&1
#       git pull >/dev/null 2>&1
#       git branch -a >/dev/null 2>&1
#       git fetch --all --prune >/dev/null 2>&1
#       popd >/dev/null 2>&1
#     done
#   fi
# }

#---------------------------------------------------------------------------------------------------------------------------
# MAIN
#---------------------------------------------------------------------------------------------------------------------------
[[ "${DEBUG}" == 'true' ]] && set -o xtrace

_pre_checks
_get_projects
_clone_projects