Bash script using GitHub Cli with GraphQl query to backup repositories.

README.md

Backup GitHub Repositories

Description: Backup all repositories belonging to a specific owner.

Author: AniMus

Before to use this script

You must have a valid account on GitHub and you have to download and install the GitHub CLI (if you need more info about this cli tool follow this link). After the installation of the GitHub CLI you can check the installation running the following command:

animus@host:~$gh --version

You should see the version of the cli like shown below (at the time of writing the last version is 1.4.0 )

gh version 1.4.0 (2020-12-15)
https://github.com/cli/cli/releases/latest

After you need to authenticate the cli running the following command to start interactive setup of your credentials:

animus@host:~$gh auth login

Follow the directions on the screen to use your GitHub credentials in the GitHub CLI (you can use either the browser or an authentication token to authenticate you).

How to use this script

Run the script without filtering

Run the script the defining the only two mandatory parameters owner and the backup directory

animus@host:~$repos.github.backup.sh -ow OWNER -dr DIRECTORY

Run the script with filtering

You can run the script filtering with custom regular expressions repositories names (e.g. ^react*.) or branches name (e.g. ^((master)|(main)|(develop))$).

animus@host:~$repos.github.backup.sh -ow OWNER -dr DIRECTORY -rr "VALID REGEX" -br "VALID REGEX" 

Run the script in help mode

Display the help running:

animus@host:~$repos.github.backup.sh --help

You'll get help

usage: repos.github.backup.sh -ow OWNER -dr DIRECTORY
Backup one or more repositories hosted on GitHub in a local directory.

Options:
-ow, --owner          Owner of repository
-dr, --directory      Backup Directory 
-rr, --reposRegex     Regular expression to match repositories names to backup (default any)
-br, --branchesRegex  Regular expression to match repositories branches to backup (default any)
-ps, --pageSize       Page size of api paging (default 10)
--help                Display this help and exit

Examples of use

Download all react repositories owned by Facebook:

repos.github.backup.sh --owner facebook -dr . -rr "^react.*" -br "^((master)|(main)|(develop))$"

Download all vscode repositories owned by Microsoft:

repos.github.backup.sh --owner microsoft -dr . -rr "^vscode..*" -br "^((master)|(main)|(develop))$"

Download all dart repositories owned by Google:

repos.github.backup.sh --owner google -dr . -rr "^dart..*" -br "^((master)|(main)|(develop))$"

repos.github.backup.sh

#!/bin/bash

#Constants for printing
SWITCH="\033["
NORMAL="${SWITCH}0m"
HEADER="${SWITCH}1;36m"
WARNING="${SWITCH}1;33m"
FAILURE="${SWITCH}1;31m"
SUCCESS="${SWITCH}1;32m"
EXTERNAL="${SWITCH}1;35m"

#GraphQl query to use to inquiry GitHub's Apis.
graphQlQuery="
query (\$owner: String!, \$pageSize: Int!, \$afterCursor: String) {
  repositoryOwner(login: \$owner) {
    repositories(first: \$pageSize, after: \$afterCursor) {                
      edges {
        node {
		      name
        }        
      }
      pageInfo{
        hasNextPage
        endCursor
      }
    }
  }
}"


##############################################################
#Default parameters
##############################################################
helpMode=false
pageSize=10
owner="" 
backupDirectory=""
home=$(pwd)
reposRegex=".*"
branchesRegex=".*" # e.g: branchesRegex="^((develop)|(master)|(main))$"
endCursor="null"
hasNextPage="true"
####################################################


##############################################################
#Functions
##############################################################
fn_sign () {
	echo -e "${HEADER}AM (aka animusna): bye bye ;)"
  echo ""
}

fn_helper () {
cat << EOF
usage: repos.github.backup.sh -ow OWNER -dr DIRECTORY
Backup one or more repositories hosted on GitHub in a local directory.

Options:
-ow, --owner          Owner of repository
-dr, --directory      Backup Directory 
-rr, --reposRegex     Regular expression to match repositories names to backup (default any)
-br, --branchesRegex  Regular expression to match repositories branches to backup (default any)
-ps, --pageSize       Page size of api paging (default 10)
--help                Display this help and exit

EOF
}

fn_readValue()
{
  ret=$(sed 's/["]//g' <<< $1 | sed 's/[^[:print:]]//g')
  echo "$ret"
}

#Option manager
fn_optionManger () {

  while [ "$#" -gt 0 ]; do
    # echo $1

    if [ $1 == "--help" ]; then
	  	helpMode=true	
    fi

    if [ $1 == "-ow" ] || [ $1 == "--owner" ]; then
      shift 1    
      owner=$1  
    fi

    if [ $1 == "-dr" ] || [ $1 == "--directory" ]; then
      shift 1    
      backupDirectory=$1  
    fi

    if [ $1 == "-rr" ] || [ $1 == "--reposRegex" ]; then
      shift 1    
      reposRegex=$1  
    fi

    if [ $1 == "-ps" ] || [ $1 == "--pageSize" ]; then
      shift 1    
      pageSize=$1  
    fi


    if [ $1 == "-br" ] || [ $1 == "--branchesRegex" ]; then
      shift 1    
      branchesRegex=$1  
    fi

    shift 1    
    ((count=count-1))
  done
  
}

fn_optionChecker(){
  errors=false

  if [ -z "$owner" ]; then
    echo -e "${FAILURE}Error: ${WARNING}owner${NORMAL} must be not empty!"
    errors=true
  fi

  if [ -z "$backupDirectory" ] || [ ! -d $backupDirectory ]; then
    echo -e "${FAILURE}Error: ${WARNING}backup directory${NORMAL} not valid or empty!"
    errors=true
  fi

  if [ $errors == true ]; then
    exit 1
  fi

}

#####################################################################################
# Start script
#####################################################################################

fn_optionManger "$@"

# Check help mode.
if $helpMode; then
	fn_helper
	fn_sign
	exit;
fi

fn_optionChecker

url="https://github.com/""$owner"

echo "**********************************************************************************************************"
echo "************************************* GitHub Repositories Backup *****************************************"
echo "**********************************************************************************************************"
echo -e "${HEADER}Owner: ${WARNING} $owner${NORMAL}!"
echo -e "${HEADER}Directory: ${WARNING} $backupDirectory${NORMAL}!"
echo -e "${HEADER}Url: ${WARNING} $url${NORMAL}!"
echo -e "${HEADER}Regex repositories: ${WARNING} $reposRegex${NORMAL}!"
echo -e "${HEADER}Regex branches: ${WARNING} $branchesRegex${NORMAL}!"
echo -e "${HEADER}Page size: ${WARNING} $pageSize${NORMAL}!"
echo "**********************************************************************************************************"
echo ""
echo ""
# echo $url

while [ $hasNextPage == "true" ]; do
  response=$(gh api graphql -F  owner=$owner -F pageSize=$pageSize -F afterCursor=$endCursor  -f query="$graphQlQuery")

  # Putting the response on more line to make easier the search of only the interested lines.
  response=$(sed 's/[{}\,]/\r\n/g' <<< $response )  
  
  # Reading values
  hasNextPage=$(fn_readValue $(grep hasNextPage <<< "$response" | sed 's/\"hasNextPage\"\://g'))
  endCursor=$(fn_readValue $(grep endCursor <<< "$response" | sed 's/\"endCursor\"\://g'))

  repos=$(grep name <<< "$response" | sed 's/\"name\"\://g' | sed 's/["]//g')
  
  cd $backupDirectory

  IFS=$'\n'
  for repo in $repos; do

    repo=$(sed 's/[^[:print:]]//g' <<< "$repo" )

    repoUrl=$url"/"$repo".git"

    repoDir=$backupDirectory"/"$repo

    if ! [[ $repo =~ $reposRegex ]]; then
      echo -e "${HEADER}Info:${NORMAL} Skipping repository ${WARNING}$repo${NORMAL}!"    	
      continue
    fi

    echo "=========================================================================================================="
    echo -e "${HEADER}Info: ${NORMAL} Backup of repository ${WARNING}$repo${NORMAL} from ${WARNING}$repoUrl${NORMAL}!"
    echo "=========================================================================================================="
    echo ""

    isGitRepo=false
    
    echo -e "${EXTERNAL}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> GIT >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>${NORMAL}"

    if ! [ -d "$repo" ]; then
      echo -e "${HEADER}Info: {NORMAL} Cloning repository ${WARNING}$repo${NORMAL} from ${WARNING}$repoUrl${NORMAL}!"    	
      git clone $repoUrl
    fi

    if [ -d "$repo" ]; then
      cd $repo

      if [ -d .git ]; then
        cd .git
        if [ "$(git rev-parse --is-inside-git-dir 2> /dev/null)" == "true" ]; then
          isGitRepo=true
        fi
        cd ..
      fi;	    
    fi

    if ! $isGitRepo; then
      echo -e "${WARNING}Warning:${NORMAL}Directory "$repoDir" found but not valid repo!"
    else

      remoteBranches=$(git branch -r | grep -v '\->' | sed 's/ //g' )

      IFS=$'\n'
      for remoteBranch in $remoteBranches; do          
        
        remoteBranchName=$(sed 's/origin\///g' <<< $remoteBranch)

        # Check regex before to track a branch.
        if [[ $remoteBranchName =~ $branchesRegex ]]; then

          # if ! [ -z "$" ]; then	

            if ! [ -z "$(git branch | grep -i $remoteBranchName$)" ]; then	
              echo -e "${HEADER}Info: ${NORMAL} ${WARNING}$remoteBranchName ${NORMAL}already tracked!"
              continue
            fi

            echo -e "${HEADER}Info: ${NORMAL}Tracking branch:"$remoteBranchName

            git branch --track $remoteBranchName
            echo -e "${HEADER}Info: ${NORMAL} ${WARNING}$remoteBranchName ${NORMAL}now tracked!"              

          # fi
        else
          echo -e "${HEADER}Info:${NORMAL} Skipping branch ${WARNING}$remoteBranchName${NORMAL}!"  
        fi
      done
      
      git pull --all
      echo -e "${EXTERNAL}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< GIT <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<${NORMAL}"
      cd ..
    fi

  done

  echo ""
  echo -e "${HEADER}Info: ${NORMAL} Next cursor ${WARNING}$endCursor${NORMAL}"
  echo -e "${HEADER}Info: ${NORMAL} Has next page: ${WARNING}$hasNextPage${NORMAL}"
  echo ""

done

cd $home

echo ""
fn_sign

exit;