Gemfile

source 'https://rubygems.org'

gem 'gooddata'
gem 'json'
gem 'pmap'

zd_massive_full_load.rb

#! /usr/bin/env ruby

require 'rubygems'
require 'gooddata'
require 'pp'
require 'json'
require 'csv'
require 'pmap'
require 'set'
require 'optparse'
require 'yaml'

DONE = 'SYNCHRONIZED'
WAITING = 'Another process is already running'
UNKNOWN = 'Another process'
TRYING = 'Trying to start'
ERROR = 'ERROR'

def read_input_csv_array(path)
  res = []
  CSV.foreach(path, :headers => false, :return_headers => false) do |row|
    res << row[0]
  end
  return res
end

def read_input_csv_hash(path)
  res = {}
  p = {}
  CSV.foreach(path, :headers => false, :return_headers => false) do |row|
    res[row[0]] = row[1]
    p[row[0]] = row[2]
  end
  return res, p
end

def main
  GoodData.logger = GoodData::NilLogger.new

  begin
    options = YAML.load_file('zd_config.yaml')
  rescue
    options = {}
  end

  OptionParser.new do |opts|
    opts.banner = "Usage: ruby zd_massive_full_load.rb [options]"
    opts.on('-u', '--username EMAIL', 'Username') { |v| options['username'] = v }
    opts.on('-p', '--password PASSWORD', 'Password') { |v| options['password'] = v }
    opts.on('-i', '--input FILE', 'Input File') { |v| options['input'] = v }
    opts.on('-o', '--output FILE', 'Output File') { |v| options['output'] = v }
  end.parse!

  client = GoodData.connect(options['username'], options['password'], {:webdav_server => "https://na1-di.gooddata.com", :server => "https://analytics.zendesk.com"})
  puts "Logged in as #{options['username']}."

  input = read_input_csv_array(options['input'])
  process, status = read_input_csv_hash(options['output'])
  output = []
  started = {}

  begin
    input.each do |pid|
      if not process.has_key?(pid) or process[pid] == WAITING # full load was not started yet
        begin
          response = client.post("/gdc/projects/#{pid}/connectors/zendesk4/integration/processes", '{"process": {"incremental": false}}')
          p = response['uri']
          process[pid] = p
          status[pid] = TRYING
        rescue RestClient::BadRequest => e # 400 Bad Request: Another process is running, cannot start full load
          process[pid] = WAITING
          response = client.get("/gdc/projects/#{pid}/connectors/zendesk4/integration")
          begin
            status[pid] = "#{UNKNOWN}: #{response['integration']['runningProcess']['status']['code']}"
          rescue
            status[pid] = UNKNOWN
          end
        rescue RestClient::Conflict => e
          process[pid] = WAITING
          status[pid] = "CONFLICT"
        end
      elsif status[pid] != DONE and status[pid] != ERROR # full load running, update status
        begin
          response = client.get(process[pid])
          s = response["process"]["status"]["code"]
          status[pid] = s
          started[pid] = response["process"]["started"]
        rescue # status unknown, have to restart the process
          process[pid] = WAITING
          status[pid] = "RESTART"
        end
      else # full load completed or failed, skip project
      end
    end
  ensure
    CSV.open(options['output'], 'w') do |outputfile|
      input.each do |pid|
        outputfile << [pid, process[pid], status[pid]]
      end
    end
  end

  status = {}
  CSV.foreach(options['output'], :headers => false, :return_headers => false) do |row|
    key = row[2]
    if status.has_key?(key)
      status[key] += 1
    else
      status[key] = 1
    end
  end
  status.each do |s|
    puts "#{s}"
  end

  puts "Downloading started at:"
  started.sort_by(&:last).each do |p|
    puts "#{p[0]} #{p[1]}"
  end
end

main()