A simple script I use to perform repetitive mechanical tasks across multiple source code repositories

xfer.rb

require 'fileutils'

def mysystem(command)
  puts "system (#{Dir.pwd}): #{command}" if @verbose
  system(command) || (raise "Error executing #{command} in #{Dir.pwd}")
end

def in_dir(dir, &block)
  begin
    Dir.chdir(dir)
    block.call
  ensure
    Dir.chdir(dir)
  end
end

def in_base_dir(&block)
  in_dir(@base_dir, &block)
end

def in_app_dir(app, &block)
  in_dir("#{@base_dir}/#{File.basename(app)}", &block)
end

COMMANDS = {}

def command(key, options = {:in_app_dir => true}, &block)
  raise "Attempting to define duplicate command #{key}" if COMMANDS[key.to_s]
  params = options.dup
  COMMANDS[key.to_s] = Proc.new do |app|
    if params[:in_app_dir]
      in_app_dir(app) do
        block.call(app)
      end
    else
      in_base_dir do
        block.call(app)
      end
    end
  end
end

def run(key, app)
  puts "Unknown command specified: #{key}" unless COMMANDS[key.to_s]
  COMMANDS[key.to_s].call(app)
end

def rbenv_exec(command)
  mysystem("unset RBENV_DIR; unset RBENV_VERSION; unset RBENV_ROOT; unset RBENV_HOOK_PATH; rbenv exec #{command}")
end

command(:clone, :in_app_dir => false) do |app|
  base_git_url = SOURCE_TREE_SET[:base_git_url]
  mysystem("git clone #{base_git_url}/#{app}.git") unless File.directory?(File.basename(app))
end

command(:fetch) do
  mysystem('git fetch --prune')
end

command(:reset) do
  mysystem('git reset --hard > /dev/null 2> /dev/null')
  mysystem('git clean -f -d -x')
end

command(:reset_origin) do
  mysystem('git reset --hard origin/master > /dev/null 2> /dev/null')
  mysystem('git clean -f -d -x')
end

command(:goto_master) do
  mysystem('git checkout master > /dev/null 2> /dev/null')
end

command(:receive_deny_current_branch) do
  mysystem('git config receive.denyCurrentBranch false')
end

command(:pull) do
  mysystem('git pull')
end

command(:push) do
  mysystem('git push origin')
end

command(:clean, :in_app_dir => false) do |app|
  run(:clone, app)
  run(:receive_deny_current_branch, app)
  run(:fetch, app)
  run(:reset, app)
  run(:goto_master, app)
  run(:pull, app)
end

command(:bundle_install) do
  rbenv_exec('bundle install') if File.exist?("#{Dir.pwd}/Gemfile")
end

command(:buildr_artifacts) do
  if File.exist?("#{Dir.pwd}/buildfile")
    begin
      rbenv_exec('buildr artifacts:sources artifacts clean')
    rescue
    end
  end
end

def patch_file(file, &block)
  filename = "#{Dir.pwd}/#{file}"
  if File.exist?(filename)
    contents = IO.read(filename)
    new_contents = block.call(contents.dup)
    if contents != new_contents
      File.open(filename,'wb') {|f| f.write new_contents}
      mysystem("git add #{file}")
      return true
    end
  end
  false
end

command(:patch_database_example_yml) do |app|
  patched = patch_file('config/setup.sh') do |content|
    content.gsub("DB_SERVER_PORT=${DB_SERVER_PORT:-1433}","DB_SERVER_PORT=${DB_SERVER_PORT:-2222}")
  end
  require 'fileutils'
  if patched
    mysystem('git commit -m "Fix the default database port."')
    puts "Cleanup setup.sh files in #{app}"
  end
end

command(:patch_suppressions_xml) do |app|
  if File.exist?('etc/checkstyle/suppressions.xml')
    patched = patch_file('etc/checkstyle/suppressions.xml') do |content|
      content.
        gsub('<suppress checks="MethodName" files=".*[\\\\/]test[\\\\/]java[\\\\/].*\.java"/>', '<suppress checks="MethodName" files="(.*[\\\\\\\\/])?test[\\\\\\\\/]java[\\\\\\\\/].*\.java"/>').
        gsub('<suppress checks="MagicNumber" files=".*[\\\\/]test[\\\\/]java[\\\\/].*\.java"/>', '<suppress checks="MagicNumber" files="(.*[\\\\\\\\/])?test[\\\\\\\\/]java[\\\\\\\\/].*\.java"/>').
        gsub('<suppress checks="ImportControl" files=".*[\\\\/]test[\\\\/]java[\\\\/].*\.java"/>', '<suppress checks="ImportControl" files="(.*[\\\\\\\\/])?test[\\\\\\\\/]java[\\\\\\\\/].*\.java"/>')
    end
    if patched
      mysystem('git commit -m "Fix suppressions for test directories in the top-level project."')
      puts "Cleanup suppressions.xml files in #{app}"
    end
  end
end

command(:patch_pmd_suppressions) do |app|
  run(:goto_master, app)

    mysystem('git commit -a -m "Remove PMD suppressions no longer required." && git push') rescue
    puts "Update the suppressions in #{app}"
end

command(:patch_iris_reports_dep) do |app|
  run(:goto_master, app)

  patched = patch_file('build.yaml') do |content|
    content.
      gsub("au.gov.vic.dse.fire.iris-reports:iris-reports:jar:8303182-13","iris.reports:iris-reports:jar:9630aed-18").
      gsub("au.gov.vic.dse.fire.iris-reports:iris-reports:jar:8731b77-14","iris.reports:iris-reports:jar:9630aed-18")
  end
  if patched
    mysystem('git commit -m "Update the iris-reports dependency."')
    puts "Update the iris-reports dependency in #{app}"
  end
end

command(:patch_mercury_dep) do |app|
  run(:goto_master, app)

  patched = patch_file('build.yaml') do |content|
    content.
      gsub("au.gov.vic.dse.fire.mercury:","mercury.messages:")
  end
  if patched
    mysystem('git commit -m "Update the group of the mercury message formats."')
    puts "Update the group of the mercury message formats in #{app}"
  end
end

command(:patch_appconfig_dep) do |app|
  run(:goto_master, app)

  patched = patch_file('build.yaml') do |content|
    content.
      gsub("iris.appconfig:iris-appconfig-db:jar:e5c5a23-30","iris.appconfig:iris-appconfig-db:jar:6b70795-37").
      gsub("iris.appconfig:iris-appconfig-server:jar:e5c5a23-30","iris.appconfig:iris-appconfig-server:jar:6b70795-37").
      gsub("iris.appconfig:iris-appconfig-qa-support:jar:e5c5a23-30","iris.appconfig:iris-appconfig-qa-support:jar:6b70795-37")
  end
  if patched
    mysystem('git commit -m "Upgrade the version of iris-appconfig in use to 6b70795-37."')
    puts "Upgraded appconfig in #{app}"
  end
end

command(:patch_syncrecord_dep) do |app|
  run(:goto_master, app)

  patched = patch_file('build.yaml') do |content|
    content.
      gsub("iris.syncrecord:iris-syncrecord-db:jar:d836251-30","iris.syncrecord:iris-syncrecord-db:jar:a2eb8cc-40").
      gsub("iris.syncrecord:iris-syncrecord-server:jar:d836251-30","iris.syncrecord:iris-syncrecord-server:jar:a2eb8cc-40").
      gsub("iris.syncrecord:iris-syncrecord-qa-support:jar:d836251-30","iris.syncrecord:iris-syncrecord-qa-support:jar:a2eb8cc-40")
  end
  if patched
    mysystem('git commit -m "Upgrade the version of syncrecord in use to a2eb8cc-40."')
    puts "Upgraded syncrecord in #{app}"
  end
end

command(:patch_clean) do |app|
  run(:goto_master, app)

  patched = patch_file('buildfile') do |content|
    content.
      gsub("  project.clean { rm_rf _('database/generated') }\n","").
      gsub("  project.clean { rm_rf \"\#{File.dirname(__FILE__)}/database/generated\" }\n","").
      gsub("  clean { rm_rf \"\#{File.dirname(__FILE__)}/database/generated\" }\n","")
  end
  if patched
    mysystem('git commit -m "Remove unneeded clean actions."')
    puts "Fixed up clean actions in buildfile for #{app}"
  end
end

command(:patch_project_clean) do |app|
  run(:goto_master, app)

  patched = patch_file('buildfile') do |content|
    content.
      gsub("  clean { rm_rf _(:target, :generated) }\n","")
  end
  if patched
    mysystem('git commit -m "Remove unnecessary clean actions."')
    puts "Fixed up clean actions in buildfile for #{app}"
  end
end

command(:patch_sil_extension) do |app|
  run(:goto_master, app)

  patched = patch_file('buildfile') do |content|
    content.gsub("require 'buildr/single_intermediate_layout'\n",'')
  end
  if patched
    mysystem('mkdir -p tasks')
    File.open('tasks/single_intermediate_layout_patch.rake','w') {|f| f.write IO.read('/Users/peter/Code/dse/skillbank/tasks/single_intermediate_layout_patch.rake')}
    mysystem('git add -f tasks/single_intermediate_layout_patch.rake')
    mysystem('git commit -m "Patch the version of the single_intermediate_layout addon in use."')
    puts "Upgraded single_intermediate_layout addon in #{app}"
  end
end

command(:patch_findbugs_extension) do |app|
  run(:goto_master, app)

  patched = patch_file('buildfile') do |content|
    content.gsub("  findbugs.config_directory = _('etc/findbugs')\n",'')
  end
  if patched || File.exist?('etc/findbugs/filter.xml')
    mysystem('git rm etc/findbugs/filter.xml && git add etc/findbugs/filter.xml') rescue
    mysystem('git commit -m "Remove over configuration of findbugs."')
    puts "Upgraded findbugs extension in #{app}"
  end
end

def braid_update(app, path)
  run(:goto_master, app)
  if File.exist?(path)
    begin
      rbenv_exec("braid update #{path}")
    rescue
      mysystem("git remote rm master/braid/#{path} >/dev/null 2>/dev/null") rescue
      rbenv_exec("braid update #{path}")
    end
    puts "Upgraded #{path} in #{app}"
  end
end

command(:braid_update_dbt) do |app|
  braid_update(app, 'vendor/plugins/dbt')
end

command(:braid_update_domgen) do |app|
  braid_update(app, 'vendor/plugins/domgen')
end

def patch_gemfile(commit_message, &block)
  filename = "#{Dir.pwd}/Gemfile"
  if File.exist?(filename)
    contents = IO.read(filename)
    new_contents = block.call(contents.dup)
    if contents != new_contents
      File.open(filename,'wb') {|f| f.write new_contents}
      mysystem('rm -f Gemfile.lock')
      rbenv_exec('bundle install')
      mysystem('git add Gemfile')
      begin
        # Not all repos have lock files checked in
        mysystem('git ls-files Gemfile.lock --error-unmatch > /dev/null 2> /dev/null && git add Gemfile.lock') 
      rescue
      end
      mysystem("git commit -m \"#{commit_message}\"")
      return true
    end
  end
  false
end

command(:patch_buildr_version) do |app|
  run(:goto_master, app)
  patched = patch_gemfile("Update to the latest version of Buildr gem.") do |content|
    content.
      gsub("gem 'buildr', '= 1.4.18'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.17'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.16'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.15'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.14'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.13'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.12'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.11'", "gem 'buildr', '= 1.4.19'").
      gsub("gem 'buildr', '= 1.4.10'", "gem 'buildr', '= 1.4.19'")
      
  end
  puts "Upgraded Buildr version in #{app}" if patched
end


command(:patch_braid_version) do |app|
  run(:goto_master, app)
  patched = patch_gemfile("Update to the latest version of braid.") do |content|
    content.gsub("gem 'realityforge-braid', '= 0.9.8'", "gem 'realityforge-braid', '= 0.9.9'")
  end
  puts "Upgraded Braid version in #{app}" if patched
end

command(:patch_tiny_tds_version) do |app|
  run(:goto_master, app)
  patched = patch_gemfile("Update to the latest version of tiny_tds.") do |content|
    content.
      gsub("gem 'tiny_tds', '= 0.6.1'", "gem 'tiny_tds', '= 0.6.2'").
      gsub("gem 'tiny_tds', '= 0.5.1'", "gem 'tiny_tds', '= 0.6.2'")
  end
  puts "Upgraded TinyTDS version in #{app}" if patched
end

command(:ensure_clean_whitespace) do |app|
  run(:goto_master, app)

  next if app == 'infrastructure'

  extensions = ['jsp', 'sass', 'scss', 'xsl', 'sql', 'haml', 'less', 'rake', 'xml', 'html', 'gemspec', 'properties', 'yml', 'yaml', 'css', 'rb', 'java', 'xhtml', 'rdoc', 'txt', 'erb', 'gitattributes', 'gitignore', 'xsd', 'textile', 'md']
  full_filenames = ['rakefile','Rakefile','buildfile','Buildfile','Gemfile','LICENSE']

  files = full_filenames.collect{|file|Dir["**/#{file}"]}.flatten + extensions.collect{|extension| Dir["**/*.#{extension}"] + Dir["**/.#{extension}"] }.flatten

  files.each do |f|
    next if /^vendor\/.*/ =~ f
    content = File.read(f)
    original_content = content.dup
    begin
      puts "Fixing DOS EOL: #{f}" if content.gsub!(/\r\n/, "\n")
      puts "Fixing Trailing whitespace: #{f}" if content.gsub!(/[ \t]+\n/, "\n")
      content.gsub!(/[ \r\t\n]+\Z/, '')
      content += "\n"
    rescue => e
      puts "Skipping whitespace cleanup: #{f}"
    end
    if content != original_content
      puts "Fixing: #{f}"
      File.open(f, 'wb') do |f|
        f.write content
      end
    end
  end

  mysystem('git reset 2> /dev/null > /dev/null')
  mysystem('git add --all --force . 2> /dev/null > /dev/null')
  begin
    mysystem('git commit -m "Cleanup whitespace at EOL and EOF." 2> /dev/null > /dev/null')
    puts "Whitespace cleaned up in #{app}"
  rescue
    # There is no need to commit if no changes made
  end
end

require 'optparse'

#@base_dir = File.expand_path(File.dirname(__FILE__))
@container_dir = File.expand_path('~/Code/SourceTree')
@base_dir = nil
@verbose = false
@quiet = false
@app_key = nil
@source_tree_set = nil

SOURCE_TREES = {
  'DEPI' => {
    :apps => %w{acal calendar irk iris planner sauron infrastructure firemod glassfish-fireweb-realm mercury gatekeeper ember iris-syncrecord iris-audit epwp rats iris-appconfig dev-template aircraft-cwn notes sfs calendar bom firetimes fbap gis-metadata-redirector awms fire-project-proposals skillbank ldar funnel iris-reports emap_rats},
    :base_git_url => 'git@git.fire.dse.vic.gov.au:fisg'
  },
  'GITHUB' => {
    :apps => %w{realityforge/guiceyloops realityforge/gwt-packetio-example realityforge/swung_weave realityforge/gwt-webpoller-example realityforge/dbt realityforge/domgen realityforge/gwt-contacts realityforge/gwt-webpoller realityforge/gwt-websockets realityforge/gwt-websockets-example realityforge/gwt-eventsource realityforge/gwt-eventsource-example realityforge/gwt-property-source realityforge/gwt-property-source-example realityforge/gwt-packetio-example realityforge/replicant realityforge/replicant-example realityforge/gwt-datatypes realityforge/simple-session-filter realityforge/gwt-cache-filter realityforge/gwt-cache-filter-example realityforge/gelf4j realityforge/gwt-appcache-example realityforge/gwt-appcache realityforge/geolatte-geom-jpa realityforge/geolatte-geom-eclipselink realityforge/dbdiff realityforge/rest-criteria realityforge/gwt-online-example realityforge/gwt-online realityforge/sqlshell realityforge/gwt-ga realityforge/getopt4j realityforge/jeo realityforge/braid realityforge/jndikit realityforge/jsyslog-message realityforge/proxy-servlet realityforge/rest-field-filter realityforge/spydle realityforge-experiments/fgis-java},
    :base_git_url => 'https://github.com'
  }
}

optparse = OptionParser.new do |opts|
  opts.on('-s', '--source-tree-set SET', 'Specify the set of projects to process') do |set|
    unless SOURCE_TREES.keys.include?(set)
      puts "Bad source tree set #{set} specified. Specify one of:\n#{SOURCE_TREES.keys.collect { |c| "  * #{c}" }.join("\n")}"
      exit
    end
    @source_tree_set = set
  end

  opts.on('-d', '--base-dir DIR', 'Specify the base directory') do |dir|
    @base_dir = File.expand_path(dir)
  end

  opts.on('--first-app APP_NAME', 'The first app to process actions for') do |app_key|
    @app_key = app_key
  end

  opts.on('-v', '--verbose', 'More verbose logging') do
    @verbose = true
    @quiet = false
  end

  opts.on('-q', '--quiet', 'Be very very quiet, we are hunting wabbits') do
    @verbose = false
    @quiet = true
  end

  opts.on('-h', '--help', 'Display this screen') do
    puts opts
    exit
  end
end

optparse.parse!

if 0 == ARGV.size
  puts "No commands specified. Specify one of:\n#{COMMANDS.keys.collect { |c| "  * #{c}" }.join("\n")}"
  exit
end

if @source_tree_set.nil?
  puts "No source tree set"
  exit
end

SOURCE_TREE_SET = SOURCE_TREES[@source_tree_set]

ARGV.each do |command|
  unless COMMANDS[command]
    puts "Unknown command specified: #{command}"
    exit
  end
end

if @verbose
  puts "Base Directory: #{@base_dir}"
  puts "Commands specified: #{ARGV.collect { |c| c.to_s }.join(', ')}"
end

@base_dir ||= "#{@container_dir}/#{SOURCE_TREE_SET[:base_dir] || @source_tree_set}"

apps = SOURCE_TREE_SET[:apps] || (raise "Unable to determine list of apps for #{@source_tree_set}")

FileUtils.mkdir_p @base_dir

skip_apps = !@app_key.nil?
apps.each do |app|
  skip_apps = false if !@app_key.nil? && @app_key == app
  if skip_apps
    puts "Skipping #{app}" if @verbose 
  else  
    puts "Processing #{app}" unless @quiet
    in_base_dir do
      ARGV.each do |key|
        run(key, app)
      end
    end
  end
end