script to clone all repos for an organisation

fetch_org_repos.rb

#!/usr/bin/env ruby

require "graphql/client"
require "graphql/client/http"
require "json"
require 'set'
require 'pp'

MAX_PAGES = 999

module GithubRepositories
  GITHUB_API_TOKEN = ENV['GITHUB_API_TOKEN']
  CACHE_REPO_DATA = ENV['CACHE_REPO_DATA'] == 'Y'

  unless GITHUB_API_TOKEN
    raise "You must set a GITHUB_API_TOKEN variable to query github"
  end

  HTTP = GraphQL::Client::HTTP.new("https://api.github.com/graphql") do
    def headers(context)
      {"Authorization" => "token #{GITHUB_API_TOKEN}"}
    end
  end

  unless File.exist? "github_schema.json"
    puts "Querying and saving github graphql schema data to github_schema.json"
    GraphQL::Client.dump_schema(HTTP, "github_schema.json")
  end

  SCHEMA = GraphQL::Client.load_schema("github_schema.json")

  CLIENT = GraphQL::Client.new(schema: SCHEMA, execute: HTTP)

  REPO_QUERY = CLIENT.parse <<-'GRAPHQL'
query($orgname: String!, $cursor: String) {
  organization(login:$orgname) {
    repositories(first:100, after: $cursor, isLocked: false) {
      pageInfo {
        endCursor
      }
      edges {
        node {
          ... on Repository {
            name
            description
            hasWikiEnabled
            isArchived
            isFork
            isPrivate
            languages(first: 100) {
              edges {
                node {
                  name
                }
              }
            }
            primaryLanguage {
              name
            }
            pushedAt
            repositoryTopics(first:100) {
              edges {
                node {
                  topic {
                    name
                  }
                }
              }
            }
            url
            sshUrl
          }
        }
      }
    }
  }
}
  GRAPHQL


  class RepositoryScanner

    def initialize(orgname)
      @orgname = orgname
    end

    def checking_query(query, variables, *properties)
      response = CLIENT.query(query, variables: variables)
      unless response.errors.empty?
        raise response.errors.inspect
      end
      rval = response.data.to_h.dig(*properties.map {|p| p.to_s})
      unless rval
        puts "can't find nested field #{properties.join(',')} in query with params:"
        pp variables
        puts "response was:"
        pp response.data.to_h
        raise "No response entry for nested keys #{properties.join(',')}"
      end
      rval
    end

    def paginated_query(query, variables, *properties)
      pages = 1
      response = checking_query(query, variables, *properties)
      edges = response["edges"]
      # response["edges"] is edges - we can concatinate them.
      unless edges
        raise "response has no edges - was query wrong?"
      end
      # response["pageInfo"] is metadata
      end_cursor = response["pageInfo"]["endCursor"]
      while end_cursor
        pages += 1
        if pages > MAX_PAGES
          $stderr.puts "bailing after #{pages} pages"
          break
        end
        vars_plus_cursor = variables.dup
        vars_plus_cursor[:cursor] = end_cursor
        response = checking_query(query, vars_plus_cursor, *properties)
        next_edges = response["edges"]
        unless next_edges
          raise "response has no edges - was query wrong?"
        end
        edges = edges + next_edges
        end_cursor = response["pageInfo"]["endCursor"]
      end
      edges.map {|e| e["node"]}
    end

    def get_all_repositories_from_github()
      paginated_query(REPO_QUERY, {orgname: @orgname}, :organization, :repositories)
    end

    def own_repositories()
      get_all_repositories_from_github.select {|r|
         r['isFork'] == false && r['isArchived'] == false
       }
    end
  end

end

if __FILE__ == $0
# code run if this is called as a script
  unless ARGV.length == 1
    raise "please specify an org"
  end
  orgname = ARGV[0]
  scanner = GithubRepositories::RepositoryScanner.new(orgname)

  # puts JSON.pretty_generate(scanner.own_repositories)
  puts "#!/bin/bash -ex"
  scanner.own_repositories.each do |r|
    name = r['name']
    sshUrl = r['sshUrl']
    url = r['url']
    puts <<~ENDS
           if [[ -d "#{name}" ]]; then
             cd "#{name}"
             git pull --rebase
             cd ..
           else
             git clone #{sshUrl} "#{name}"
           fi
        ENDS
  end
end