Generate a bunch of JSON files from GithubArchive

githubarchive.sh

 #! /bin/sh

# requires that BigQuery CLI tools are installed
# http://code.google.com/p/google-bigquery-tools/


echo Running Query to grab most popular languages
bq --format=csv --quiet query 'SELECT TOP(repository_language, 10), count(*) FROM [githubarchive:github.timeline]' > tmpLanguages.csv

echo Removing top two lines
awk '{if (NR>2) {print}}' tmpLanguages.csv > ttmpLanguages.csv

echo Extracting languages
awk -F "\"*,\"*" '{print $1}' ttmpLanguages.csv > tttmpLanguages.csv

languages=( `cat tttmpLanguages.csv | tr '\n' ' '` )


for language in ${languages[@]} 
do
echo "Running query for $language"

bq --format=prettyjson query "SELECT repository_name, count(repository_name) as pushes, repository_description, repository_url FROM [githubarchive:github.timeline] WHERE type='PushEvent' AND repository_language='$language' GROUP BY repository_name, repository_description, repository_url ORDER BY pushes DESC LIMIT 20" > $language.json

done