stashToGitHub_RepoMapAndCopy_v3.1_anonymized.sh

#!/bin/bash

# Step 1: Auth to GitHub
# 1 Run linux / OSX w/ these dependencies installed: git, LFS, BFG, curl, github CLI
# 2. Make sure to export GITHUB_TOKEN - used to create repo and OATH for repo push

# Export
# STASH_USER='your.user'
# STASH_PASS='your_password_for_stash'
# STASH_SERVER='your.stash.server.com'

# Set new line as separator for reading data in for loops
IFS=$'\n'

# By default, runs against the "project_keys" file in this repo, which contains one project key (e.g, CON) per line
# Uncomment this curn to run against all projects in stash
#curl -s --user $STASH_USER:$STASH_PASS 'https://$STASH_SERVER/rest/api/1.0/projects?limit=1000' | jq -r .values[].key > ./projects_keys

# Writes all project info to local file in raw json format
curl -s --user $STASH_USER:$STASH_PASS 'https://$STASH_SERVER/rest/api/1.0/projects?limit=1000' > ./projects_raw

# This is the GitHub project to use
github_project_name=your-github-organization-name

for project in `cat ./projects_keys`; do
    # Gather project info
    project_key=$(cat ./projects_raw | jq --raw-output ".values[] | select(.key==\"$project\") | .key")
    project_name=$(cat ./projects_raw | jq --raw-output ".values[] | select(.key==\"$project\") | .name")
    project_url=$(cat ./projects_raw | jq --raw-output ".values[] | select(.key==\"$project\") | .links.self[].href")

    # Write out project name and URL
    echo "**********************"
    echo "Project name:" $project_name "-" $project_url
    echo "**********************"

    # Gather all repos in project
    curl -s --user $STASH_USER:$STASH_PASS https://$STASH_SERVER/rest/api/1.0/projects/$project/repos?limit=1000 > project_repos_raw

    # Loop through repos, set naming
    for repo in `cat project_repos_raw | jq -r .values[].name`; do
        # Format project name into title case
        #projectTitleCase=$(echo $project | tr '[A-Z]' '[a-z]' | awk '{for(i=1;i<=NF;i++){ $i=toupper(substr($i,1,1)) substr($i,2) }}1')
        #newRepoName=$projectTitleCase$repo

        # Prep Github repo name with hyphens instead of spaces
        github_repo_name="${repo// /-}"

        # Echo out old and new names
        echo "**********************"
        echo "Old repo path:" $project/$repo", new repo path: $github_project_name/$github_repo_name"
        echo "**********************"
    
        # Create repo in GitHub
        echo "N" | gh repo create $github_project_name/$github_repo_name --confirm --private
        
        # Clean up unwanted init'd repo from gh CLI
        rm -rf $github_repo_name

        # Grab all repo clone values
        getRepoCloneValues=$(cat project_repos_raw | jq --raw-output --arg repo "$repo" '
            .values[]
            | select( .name | contains($repo)).links.clone[]
            | select( .name | contains("ssh")) | .href')

        # Select first repo clone value if there are multiple
        IFS=';' read -r stashRepoUrl string <<< "$getRepoCloneValues"

        # Clone old repo
        git clone --mirror $stashRepoUrl $repo

        # Change context into folder
        cd $repo

        # Attempt push trap results
        echo "Attempting git push --mirror without any modifications"
        gitPushResults=
        gitPushResults=$(git push --mirror https://oauth2:$GITHUB_TOKEN@github.com/$github_project_name/$github_repo_name.git --porcelain 2>&1)
        
        # Check git exit code, and engage special transforms if required
        if [[ $? -ne 0 ]]; then

            # Check if file size error
            if echo "$gitPushResults" | grep -q "this exceeds GitHub's file size limit of 100.00 MB"; then
                
                # Define array to hold filename values
                tooLargeFilesArray=()

                # Iterate through lines, populate array
                for r in $(echo "$gitPushResults" | grep "this exceeds GitHub's file size limit of 100.00 MB" | cut -d " " -f 4); do
                    # Echo for debugging
                    #echo "File to protect: $r"

                    # Add filename to array
                    tooLargeFilesArray+=($r)
                done

                # Echo for debugging                
                #echo ${tooLargeFilesArray[@]}

                # Loop through all too-large files and use BFD to rewrite history for LFS
                echo "Loop through all too-large files and use BFD to rewrite history for LFS"
                for tooLargeFile in "${tooLargeFilesArray[@]}"; do
                    # Strip any trailing spaces
                    tooLargeFile=$(echo "$tooLargeFile" | xargs)

                    # Remove file path
                    tooLargeFile=$(echo $tooLargeFile | rev | cut -d "/" -f 1 | rev)

                    echo "Operating BFG on file" $tooLargeFile
                    bfg --convert-to-git-lfs $tooLargeFile --no-blob-protection
                done

                # Expiring git reflog, installing LFS, attempting push again
                echo "Expiring git reflog, installing LFS, attempting push again"
                git reflog expire --expire=now --all && git gc --prune=now --aggressive
                git lfs install
                
            # End file size error section
            fi

            # Check if tag name 
            if echo "$gitPushResults" | grep -q "Sorry, branch or tag names consisting of 40 hex characters are not allowed."; then
                
                # Define array to hold filename values
                badCommitsArray=()

                # Iterate through lines, populate array
                for r in $(echo "$gitPushResults" | grep "Invalid branch or tag name" | cut -d " " -f 8); do
                    # Echo for debugging
                    #echo "File to protect: $r"

                    # Add filename to array
                    badCommitsArray+=($r)
                done

                # Echo for debugging                
                #echo ${badCommitsArray[@]}

                # Loop through all too-large files and use BFD to rewrite history for LFS
                echo "Loop through all invalid commits and use git to alias new valid tag names"
                for badCommit in "${badCommitsArray[@]}"; do
                    # Remove leading and trailing quotes
                    badCommit=$(sed -e 's/^"//' -e 's/"$//' <<<"$badCommit")

                    # Create new alias for old bad commit
                    badCommitAlias=${badCommit::${#badCommit}-2}

                    # Print changes
                    echo "Old invalid commit:" $badCommit
                    echo "New updated commit, strip last 2 characters:" $badCommitAlias
                    
                    # Map alias of new --> old. Note new is exact same as old, but last 2 characters stripped
                    git tag $badCommitAlias $badCommit
                    # Delete old git tag
                    git tag -d $badCommit
                done

            # End invalid commit section
            fi

            # Try push again
            echo "Attempting git push --mirror again after modifications"
            git push --mirror https://oauth2:$GITHUB_TOKEN@github.com/$github_project_name/$github_repo_name.git

            # Check if git succeeded
            if [[ $? -eq 0 ]]; then
                echo "Modified git push succeeded"
            else
                echo "Something still wrong, need to investigate"
            fi
        else
            echo "Git push succeeded with no issues"
        fi

        # Reset to original location
        cd ..

        # Garbage collect downloaded code folder to avoid build-up
        echo "Cleaning up slush files"
        rm -rf $repo
        rm -rf *.bfg-report
    done

    echo "**********************"
    echo ""
done
unset IFS