clone_all repos

README.md

Clone all repos

Install as follows

\curl -sSL https://gist.github.com/bf4/c19f041bb0887476dfaa/download | \
  tar xzvf - --include '*setup' -O | bash

optionally first set PROJECT_DIR and PROJECT_CLONE_URL in your ~/.profile

e.g.

export PROJECT_DIR=$HOME/projects
export PROJECT_CLONE_URL=https://gist.github.com/c19f041bb0887476dfaa.git

client.rb

require 'json'
class CloneAll
  class Client
    HOST = "https://api.github.com"
    attr_reader :organization
    def initialize(oauth_token, organization)
      @oauth_token = oauth_token
      @organization = organization
    end

    def client_credentials
      { :access_token => @oauth_token }
    end

    def teams
      hit_endpoint("#{HOST}/orgs/#{organization}/teams")
    end

    def team_repos
      teams.each do |team|
        team_name = team['name']
        url = team['repositories_url']
        team_repos = hit_endpoint(url) do |http_code, json_response|
          warn "Could not get repos for #{team_name}, #{url}"\
            "\n\tGot #{http_code}, #{json_response}"
          :failure
        end
        next if team_repos == :failure
        yield(team_name, team_repos)
      end
    end

    # attributes: name
    def repos
      (1..25).each do |i|
        json = hit_endpoint("#{HOST}/orgs/#{organization}/repos?page=#{i}") do |http_code, json_response|
          fail "Code=#{http_code}, Message='Failed to get repos', Response='#{JSON.pretty_generate(json_response)}'"
        end
        log "Page #{i}"
        break if json.empty?
        yield json
      end
    end

    # https://api.github.com/emojis
    require 'cgi'
    Repo = Struct.new(:repo_name, :owner, :client) do

      def json_params(hash)
        "-H 'Content-Type: application/json' -d '#{hash.to_json}'"
      end

      def url_param(string)
        CGI.escape(string).gsub('+', '%20')
      end
      # https://developer.github.com/v3/issues/labels/
      # @return [Array<{url:,name:,color:}>]
      def labels
        client.hit_endpoint("#{HOST}/repos/#{owner}/#{repo_name}/labels")
      end

      def collaborators
        client.hit_endpoint("#{HOST}/repos/#{owner}/#{repo_name}/collaborators")
      end

      # @param label_name [String]
      # @param hex_color [String] 6 character hex code, without the leading #, identifying the color.
      # @return 201 success
      def create_label(label_name, hex_color)
        client.hit_endpoint("-XPOST #{HOST}/repos/#{owner}/#{repo_name}/labels #{json_params(name: label_name, color: hex_color)}") do |http_code, json_response|
          case http_code
          when 201 then json_response
          when 422 then log "Expected to create label #{label_name}, but got #{http_code}, #{json_response}"
          else
            fail "Expected to create label #{label_name}, but got #{http_code}, #{json_response}"
          end
        end
      end

      # @param label_name [String]
      # @param hex_color [String] 6 character hex code, without the leading #, identifying the color.
      # @return 200 success
      def update_label(label_name, hex_color)
        client.hit_endpoint("-XPATCH #{HOST}/repos/#{owner}/#{repo_name}/labels/#{url_param label_name} #{json_params(name: label_name, color: hex_color)}") do |http_code, json_response|
          if http_code != 200
            fail "Expected to update label #{label_name}, but got #{http_code}, #{json_response}"
          end
          json_response
        end
      end

      # @param label_name [String]
      # @return 204 success
      def delete_label(label_name)
        label_name = url_param(label_name)
        client.hit_endpoint("-XDELETE #{HOST}/repos/#{owner}/#{repo_name}/labels/#{label_name}") do |http_code, json_response|
          case http_code
          when 204 then json_response
          when 404 then log "Expected to delete label #{label_name}, but got #{http_code}, #{json_response}"
          else
            fail "Expected to delete label #{label_name}, but got #{http_code}, #{json_response}"
          end
        end
      end

      def debug(msg="")
        p msg
      end

      def log(msg="")
        STDOUT.puts msg
      end

      def warn(msg="")
        STDERR.puts msg
      end
    end

    def repo(repo_name)
      Repo.new(repo_name, organization, self)
    end

    HTTP_SUCCESS = (200..299)
    def hit_endpoint(url)
      auth_header = !@oauth_token.empty? && %(-H "Authorization: token #{@oauth_token}")
      cmd = <<-CMD
curl #{url} --silent \
#{auth_header} \
-H "User-Agent: CloneAllrb" \
--write-out '%{http_code}' # appends status code to stdout
      CMD
      response = `#{cmd}`
      http_code = 200
      response = response.sub(/\d{3}\z/) { |code|
        http_code = code.to_i
        ''
      }
      json = Array(JSON.load(response))
      if !HTTP_SUCCESS.cover?(http_code)
        block_given? ?
          json = yield(http_code, json) :
          fail("Failed #{url} with #{http_code}, #{json.inspect}")
      end
      json
    end

    def debug(msg="")
      p msg
    end

    def log(msg="")
      STDOUT.puts msg
    end

    def warn(msg="")
      STDERR.puts msg
    end
  end
end

clone_all.example.yml

---
access_token:    ""
organization:    ""

clone_all.rb

#!/usr/bin/env ruby
# Required: access_token
# Required: organization
# ruby clone_all.rb --repos
# ruby clone_all.rb --teams
# ruby clone_all.rb --labels template_repo
# or via clone_all.yml
# ---
# access_token:    "githuboauthtoken"
# organization:    "lolmgkittens"
# Note: repo data is cached in repos.json
require 'json'
require 'optparse'
require 'open3'
require 'pathname'
file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
dir = Pathname File.expand_path("..", file)
require dir.join('client')
require dir.join('cloner')
class CloneAll
  class Config
    attr_reader :label_source_repo, :update_all_repos, :repo_collabs, :command

    def initialize(options)
      @update_repos_list = options.fetch(:update_repos_list, false)
      @update_all_repos = options.fetch(:update_all_repos, false)

      @label_source_repo = ENV.fetch('label_source_repo') { options[:label_source_repo] }
      @clone_repos = options.fetch(:clone_repos, false)
      @repo_collabs = options.fetch(:repo_collabs, false)
      @repo_teams   = options.fetch(:repo_teams, false)
      @command = case
                 when @label_source_repo then :clone_labels
                 when @clone_repos       then :clone_repos
                 when @repo_collabs      then :print_collabs
                 when @repo_teams        then :print_teams
                 else
                   :help
                 end
    end
  end

  def initialize(args=ARGV)
    trap("INT") { log "Got interrupt, time to quit"; exit 1 }
    STDOUT.sync = true
    STDERR.sync = true
    configure(args)
  end

  def configure(args)
    config = load_config
    organization = config.fetch('organization')
    oauth_token = config.fetch('access_token')
    @client = Client.new(oauth_token, organization)
    parse_options(args)
  end

  def parse_options(args)
    options = {}
    options_parser = OptionParser.new do |parser|
      parser.on('--update-repos-list', 'Hit GitHub api to update our list of repos') do |bool|
        options[:update_repos_list] = bool
      end
      parser.on('--update-all-repos', 'Update all downloaded repos from the remote and bundle (if possible)') do |bool|
        options[:update_all_repos] = bool
      end
      parser.on('-l', '--labels NAME', 'clone labels from the named repo') do |name|
        options[:label_source_repo] = name
      end
      parser.on('-r', '--repos', 'clone repos') do |bool|
        options[:clone_repos] = bool
      end
      parser.on('--collabs REPO', 'Get collaborators for the org/repo') do |repo|
        options[:repo_collabs] = repo
      end
      parser.on('--teams', 'Get teams for the org') do |repo|
        options[:repo_teams] = repo
      end
    end
    begin
      options_parser.parse!(ARGV)
      @config = Config.new(options)
    rescue ArgumentError, OptionParser::ParseError => e
      log e.message
      exit(1)
    end
  end

  def help
    log "Required options are --repos, --labels NAME, --collabs REPO, or --teams"
  end

  def run
    Cloner.new(client, config).public_method(config.command).call
  end

  private

  attr_reader :client, :config, :command

  def log(msg="")
    STDOUT.puts msg
  end

  def load_config
    require 'yaml'
    YAML.load_file('clone_all.yml')
  end


end

if $0 == __FILE__
  clone_all = CloneAll.new
  clone_all.run
end

cloner.rb

class CloneAll
  class Cloner

    def initialize(client, config)
      @client = client
      @config = config
    end

    # Clone labels from a repo
    def clone_labels(label_source_repo = config.label_source_repo)
      labels = Labels.new client.repo(label_source_repo).labels
      debug labels.names
      client.repos do |json|
        json.each do |repo_json|
          name = repo_json['name']
          next if name == label_source_repo
          repo = client.repo(name)
          repo_labels = Labels.new repo.labels
          next if repo_labels == labels
          log
          new_labels = labels - repo_labels
          if ! new_labels.empty?
            log "[#{name}] Adding #{new_labels.map(&:name)}"
            new_labels.each {|rl| repo.create_label(rl.name, rl.color) }
          end
          pruned_labels = repo_labels - labels
          if ! pruned_labels.empty?
            log "[#{name}] Removing #{pruned_labels.map(&:name)}"
            pruned_labels.each {|rl| repo.delete_label(rl.name) }
          end
        end
      end
    end

    def clone_repos(update_all_repos = config.update_all_repos)
      download_repos(client.client_credentials)

      each_repo do |repo|
        name      = repo['name']
        # clone_url = repo['clone_url']
        private_repo = repo['private'] == true
        full_name = repo['full_name']
        if clone_repo(name, full_name, private_repo) || update_all_repos
          update_repo(name)
        end
      end
    end

    def print_collabs(repo_collabs = config.repo_collabs)
      client.repo(repo_collabs).collaborators.each do |collab|
        p collab["login"]
      end
    end

    # Print out team names and member repos
    def print_teams
      require 'pp'
      client.team_repos do |team, repos|
        pp [team, repos.map {|repo| repo['name'] }]
      end
    end

    def each_repo(&block)
      pids = []
      downloaded_repos.each do |repo|
        pids << fork do
          block.call(repo)
        end
      end
      pids.each do |pid|
        Process.waitpid2(pid)
      end
    end

    def download_repos(client_credentials, client_options={:type => 'all'})
      return unless download_repos?
      @json = []
      client.repos do |json|
        names = json.map{|item| item['name'] }
        old_names =  @json.map{|item| item['name'] }
        new_names = names - old_names
        log "Got: #{new_names}\tHad: #{old_names}"
        @json = @json | json
      end
    ensure
      # if JSON parsing fails, just write what we have
      save_repos(@json)
    end

    private

    attr_reader :client, :config

    def folder_name
      File.basename Dir.pwd
    end

    def sh(command)
      # system(command << " &> /dev/null")
      captured_output = ''
      captured_errors = ''
      status = :not_set
      name = folder_name
      Open3.popen3(command) do |stdin, stdout, stderr, wait_thr|
        captured_output << stdout.read.chomp
        captured_errors << stderr.read.chomp
        status = wait_thr.value
      end
      errors = captured_output.split("\n").grep(/error|fail/i)
      no_errors = true
      if errors.size > 0
        no_errors = false
        log "errors from #{name}"
        log "\t" << errors.join("\n\t")
      end
      if captured_errors.size > 0
        message = captured_errors.split("\n").first
        if message !~ /Already on 'master'/
          no_errors = false
          log "\t ERRORS #{name.upcase}: #{message}"
        end
      end
      status.success?
    end

    def clone_repo(name, full_name, private_repo = false)
      # require 'fileutils'
      # FileUtils.rm_rf(name) if File.exist?(name)
      # return false
      return false if File.exist?(name)
      ssh_url = private_repo ? "git@github.com:#{full_name}.git" : "https://github.com/#{full_name}.git"
      clone_cmd = "git clone --recursive #{ssh_url}"
      if sh(clone_cmd)
        log "Cloned #{name}"
        true
      else
        warn "Clone failed. cmd: #{clone_cmd}"
        false
      end
    end

    def update_master(name)
      if sh("git diff --quiet head &>/dev/null")
        dep1 = sh("git branch -u origin/master")
        dep2 = sh("git checkout master && git pull --rebase") # status
        dep1 && dep2
      else
        warn "Not updating; #{name} is dirty"
        false
      end
    end

    def install_deps
      if File.exist?("Gemfile")
        sh("bundle check || bundle")
      else
        true
      end
    end

    def update_repo(name)
      Dir.chdir(name) do
        dep1 = update_master(name)
        dep2 = install_deps
        if dep1 && dep2
          log "Updated #{name}"
        end
      end
    end

    def download_repos?
      @update_repos_list || repos_file
    end

    def save_repos(json)
      return if json.nil?
      downloaded_json = downloaded_repos
      json = json.concat(downloaded_json).uniq {|element| element['full_name'] }
      log "Added #{json.size - downloaded_json.size} repos. Total: #{json.size}"
      File.write(repos_file, JSON.pretty_generate(json))
    end

    def repos_file
      !File.exist?('repos.json') && File.write('repos.json', [])
      'repos.json'
    end

    def downloaded_repos
      Array(JSON.load(File.binread(repos_file)))
    end

    def debug(msg="")
      p msg
    end

    def log(msg="")
      STDOUT.puts msg
    end

    def warn(msg="")
      STDERR.puts msg
    end
  end

  Labels = Struct.new(:labels) do
    include Comparable
    def <=>(other)
      labels == other.labels
    end
    Label = Struct.new(:label) do
      include Comparable
      def <=>(other)
        name == other.name
      end
      def name;  label['name'];  end
      def url ;  label['url'];   end
      def color; label['color']; end
    end
    def labels
      @labels ||= self[:labels].map {|label| Label.new(label) }
    end
    def names;  labels.map(&:name);  end
    def urls;   labels.map(&:url);   end
    def colors; labels.map(&:color); end
    def -(other)
      other_names = other.names
      labels.reject {|label| other_names.include?(label.name) }
    end
    def ==(other)
      names == other.names
    end
  end
end

setup

#!/usr/bin/env bash

PROJECT_DIR=${PROJECT_DIR:-$HOME/projects}
PROJECT_CLONE_URL=${CLONE_URL:-https://gist.github.com/c19f041bb0887476dfaa.git}

mkdir -p $PROJECT_DIR
cd $PROJECT_DIR && \
git clone $PROJECT_CLONE_URL clone_all && \
ln -s $PROJECT_DIR/clone_all/clone_all.rb $PROJECT_DIR/clone_all.rb
cp $PROJECT_DIR/clone_all/clone_all.example.yml $PROJECT_DIR/clone_all.yml
echo "Go to https://github.com/settings/applications"
echo "Generate a new personal access token with 'repo' priviliges"
echo "Add the access_token to ${PROJECT_DIR}/clone_all.yml"
echo "Edit ${PROJECT_DIR}/clone_all.yml as desired"
echo "run `ruby clone_all.rb` and profit"