simple and easy github gist backup

github_backup_gists.sh

#!/usr/bin/env bash

for url in `./githubapi-get.sh $GIST_TOKEN /gists | jq -r '.[]|.git_pull_url'`
do
    git clone ${url};
done

githubapi-get-all-repos.sh

#!/bin/bash

if [ ${#@} -lt 2 ]; then
    echo "usage: ./githubapi-get-all-repos.sh [github org] [your github token] --no-forks"
    exit 1;
fi

GITHUB_ORG=$1
GITHUB_TOKEN=$2
NO_FORKS=`if [ "$3" = "--no-forks" ]; then echo "1"; fi`

temp=`basename $0`
TMP_FILE_DIR=`mktemp -d /tmp/${temp}.XXXXXX` || exit 1

./githubapi-get.sh $GIST_TOKEN ${GITHUB_ORG} > $TMP_FILE_DIR/all_repos.json

if [ "$NO_FORKS" = "1" ]; then
    cat $TMP_FILE_DIR/all_repos.json | jq '[ .[] | if .fork then empty else .name end ] | sort' > $TMP_FILE_DIR/result.json
else
    cat $TMP_FILE_DIR/all_repos.json | jq '[ .[] | .name ] | sort' > $TMP_FILE_DIR/result.json
fi

cat $TMP_FILE_DIR/result.json | jq '.[]' | sed -e 's/"//g' > $TMP_FILE_DIR/result.out
cat $TMP_FILE_DIR/result.out

rm -rf $TMP_FILE_DIR

githubapi-get.sh

#!/bin/bash

set -e

if [ ${#@} -lt 2 ]; then
    echo "usage: $0 [your github token] [REST expression]"
    exit 1;
fi

GITHUB_TOKEN=$1
GITHUB_API_REST=$2

# TODO: we have to allow for passing new/custom Accept header-s form the cmd line,
#       and/or setup a lookup table of accept headers from the known headers?
#GITHUB_API_HEADER_ACCEPT="Accept: application/vnd.github.v3+json"
#GITHUB_API_HEADER_ACCEPT="Accept: application/vnd.github.sersi-preview+json"
GITHUB_API_HEADER_ACCEPT="Accept: application/vnd.github.drax-preview+json"

temp=`basename $0`
TMPFILE=`mktemp /tmp/${temp}.XXXXXX` || exit 1

# single page result-s (no pagination), have no Link: section, the grep result is empty
last_page=`curl -s -I "https://api.github.com${GITHUB_API_REST}" -H "${GITHUB_API_HEADER_ACCEPT}" -H "Authorization: token $GITHUB_TOKEN" | grep '^Link:' | sed -e 's/^Link:.*page=//g' -e 's/>.*$//g'`

# does this result use pagination?
if [ -z "$last_page" ]; then
    # no - this result has only one page
    curl -s "https://api.github.com${GITHUB_API_REST}" -H "${GITHUB_API_HEADER_ACCEPT}" -H "Authorization: token $GITHUB_TOKEN" >> $TMPFILE
    cat $TMPFILE

else
    # yes - this result is on multiple pages
    for p in `seq 1 $last_page`; do
	curl -s "https://api.github.com${GITHUB_API_REST}?page=$p" -H "${GITHUB_API_HEADER_ACCEPT}" -H "Authorization: token $GITHUB_TOKEN" | sed -e 's/^\[$//g' -e 's/^\]$/,/g' >> $TMPFILE
    done

    # return the multipage JSON result-s as a JSON array
    line_counter=`wc -l $TMPFILE | sed -e 's/[/a-zA-Z].*$//g'`

    echo "["
    head -n $(($line_counter - 1)) $TMPFILE
    echo "]"

fi

notes.md

Intro

1. create a github token (gist token or other token with enough permissions)
2. set/export the github/gist token GIST_TOKEN env var
3. run the script

Example 00

1. export GIST_TOKEN=...
2. ./github_backup_gists.sh

Example 01

1. export GIST_TOKEN=...
2. ./githubapi-get-all-repos.sh /users/mbohun/repos $GIST_TOKEN

   ALA4R
   BioLink_test
   Free-Social-Icons
   JSON.sh
   NAQS_tblBiota
   QRBWS
   RegioSQM
   SMILES.js
   ag-bie
   ala-cas
   ala-cas-2.0
   ala-cas-client
   ala-datamob
   ala-datamob_TEST
   ala-dataquality
   ala-downloads
   ala-expert
   ala-hub
   ala-install
   ala-logger-service
   ala-name-matching
   ala-names-generator
   ala-web-theme
   ala_biocache_test
   alerts
   analysis-service
   apikey
   appd-hub
   bhl-ftindex-manage
   bie-webapp2
   biocache-hubs
   biocache-store
   biolink
   blog-00
   book_library_example
   cas-strap-sample
   crawler4j
   dashboard
   doctoc
   ecodata
   emacs
   fieldcapture
   fieldcapture-mobile-android
   fielddata-mobile-android
   generic-hub
   gfx
   gtk_shader_ide
   jasig-cas-4.0.x-BUG-cas-server-webapp-support
   jasig-cas-4.0.x-overlay-template
   jasig-cas-4.1.0-overlay-template
   java-jwt
   jq
   js-yaml
   just_test
   layers-service
   linux
   mapper
   markdown-here
   mbohun.github.io
   mbohun_graph-experiments
   mgmt
   misc
   molecules
   mpc
   nsl-domain-plugin
   oracle-blog-examples
   ozcam-hub
   pac4j
   regions
   rlm_ftress
   spatial-logger
   spatial-portal
   specieslist-webapp
   specimenbrowser
   stream-handbook
   taxon-overflow
   test-ALA4R
   test-repo-one
   test-repo-two
   test-repo_hg-migrated_from-bb
   test-travis-repo
   travis-build-configuration
   trinity
   tviewer
   userdetails
   virt-test
   volunteer-portal
   whatever
   wikigraph
   

3. ./githubapi-get-all-repos.sh /users/mbohun/repos $GIST_TOKEN --no-forks

   BioLink_test
   NAQS_tblBiota
   SMILES.js
   ala-cas-2.0
   ala-datamob
   ala-datamob_TEST
   ala-dataquality
   ala_biocache_test
   biolink
   blog-00
   book_library_example
   emacs
   gfx
   gtk_shader_ide
   jasig-cas-4.0.x-BUG-cas-server-webapp-support
   jasig-cas-4.0.x-overlay-template
   jasig-cas-4.1.0-overlay-template
   just_test
   linux
   mbohun.github.io
   mbohun_graph-experiments
   misc
   molecules
   rlm_ftress
   test-ALA4R
   test-repo-one
   test-repo-two
   test-repo_hg-migrated_from-bb
   test-travis-repo
   whatever
   

REFERENCES

• https://developer.github.com/v3/
• TODO