Created
April 6, 2023 13:50
-
-
Save HalisCz/d1203ee40337d246668254b71d8b64cb to your computer and use it in GitHub Desktop.
Bash script to get list of all repos from GitHub organization and clone them in parallel. Requires bash, >=git-2.26, jq, parallel, find, sed. Also contains commented line for host substitution, in case you are using different hostname to utilize settings from .ssh/config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Modified version of https://gist.github.com/caniszczyk/3856584#gistcomment-3157288 | |
# Source https://gist.github.com/caniszczyk/3856584?permalink_comment_id=3335247#gistcomment-3335247 | |
# Sample syntax: | |
# foobar\.git | |
# foobar\.git|helloworld\.git | |
# Bash strict mode | |
set -euo pipefail | |
IFS=$'\n\t' | |
IGNORED_REPO="" | |
IGNORED_REPO_COUNT=0 | |
ORG="" | |
# per_page maxes out at 100 | |
PER_PAGE=100 | |
REPO_TYPE="all" | |
# https://github.com/settings/tokens | |
TOKEN="" | |
TEST_RUN=false | |
for ((PAGE=1; ; PAGE+=1)); do | |
# Page 0 and 1 are the same | |
# Change authorization method as needed | |
INPUT=$(curl -H "Authorization: token $TOKEN" -s "https://api.github.com/orgs/$ORG/repos?type=$REPO_TYPE&per_page=$PER_PAGE&page=$PAGE" | jq -r ".[].ssh_url") | |
if [[ -z "$INPUT" ]]; then | |
echo "All repos processed, ignored $IGNORED_REPO_COUNT repo(s) and stopped at page=$PAGE" | |
exit | |
fi | |
while read -r REPO_URL ; do | |
if [[ "$REPO_URL" =~ $IGNORED_REPO ]]; then | |
echo "*** IGNORING $REPO_URL" | |
IGNORED_REPO_COUNT=$((IGNORED_REPO_COUNT+1)) | |
else | |
# rewrite host in case you are using ssh config for different ssh key for your work account | |
#REPO_URL=${REPO_URL/github.com/github.com-ds} | |
if $TEST_RUN; then | |
echo "git clone $REPO_URL" | |
else | |
#git clone "$REPO_URL" >/dev/null 2>&1 || # Pipe stdout and stderr to /dev/null | |
#git clone --depth 1 "$REPO_URL" || # Shallow clone for faster cloning, within the repo use the following to get the full git history: git pull --unshallow | |
git clone "$REPO_URL" || # Vanilla | |
{ echo "ERROR: Unable to clone $REPO_URL!" ; continue ; } | |
fi | |
fi | |
# This syntax works as well /shrug | |
# done <<< "$INPUT" | |
done < <(echo "$INPUT") | |
done | |
# fetch all, try to merge current branch if straight forward | |
find . -type d -name ".git" |\ | |
sed 's/\/.git//g' |\ | |
parallel -j 16 --progress \ | |
"echo {}; git -C {} fetch --all; git -C {} merge --ff-only" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment