Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
source 'https://rubygems.org'
gem 'gooddata'
gem 'json'
gem 'pmap'
#! /usr/bin/env ruby
require 'rubygems'
require 'gooddata'
require 'pp'
require 'json'
require 'csv'
require 'pmap'
require 'set'
require 'optparse'
require 'yaml'
DONE = 'SYNCHRONIZED'
WAITING = 'Another process is already running'
UNKNOWN = 'Another process'
TRYING = 'Trying to start'
ERROR = 'ERROR'
def read_input_csv_array(path)
res = []
CSV.foreach(path, :headers => false, :return_headers => false) do |row|
res << row[0]
end
return res
end
def read_input_csv_hash(path)
res = {}
p = {}
CSV.foreach(path, :headers => false, :return_headers => false) do |row|
res[row[0]] = row[1]
p[row[0]] = row[2]
end
return res, p
end
def main
GoodData.logger = GoodData::NilLogger.new
begin
options = YAML.load_file('zd_config.yaml')
rescue
options = {}
end
OptionParser.new do |opts|
opts.banner = "Usage: ruby zd_massive_full_load.rb [options]"
opts.on('-u', '--username EMAIL', 'Username') { |v| options['username'] = v }
opts.on('-p', '--password PASSWORD', 'Password') { |v| options['password'] = v }
opts.on('-i', '--input FILE', 'Input File') { |v| options['input'] = v }
opts.on('-o', '--output FILE', 'Output File') { |v| options['output'] = v }
end.parse!
client = GoodData.connect(options['username'], options['password'], {:webdav_server => "https://na1-di.gooddata.com", :server => "https://analytics.zendesk.com"})
puts "Logged in as #{options['username']}."
input = read_input_csv_array(options['input'])
process, status = read_input_csv_hash(options['output'])
output = []
started = {}
begin
input.each do |pid|
if not process.has_key?(pid) or process[pid] == WAITING # full load was not started yet
begin
response = client.post("/gdc/projects/#{pid}/connectors/zendesk4/integration/processes", '{"process": {"incremental": false}}')
p = response['uri']
process[pid] = p
status[pid] = TRYING
rescue RestClient::BadRequest => e # 400 Bad Request: Another process is running, cannot start full load
process[pid] = WAITING
response = client.get("/gdc/projects/#{pid}/connectors/zendesk4/integration")
begin
status[pid] = "#{UNKNOWN}: #{response['integration']['runningProcess']['status']['code']}"
rescue
status[pid] = UNKNOWN
end
rescue RestClient::Conflict => e
process[pid] = WAITING
status[pid] = "CONFLICT"
end
elsif status[pid] != DONE and status[pid] != ERROR # full load running, update status
begin
response = client.get(process[pid])
s = response["process"]["status"]["code"]
status[pid] = s
started[pid] = response["process"]["started"]
rescue # status unknown, have to restart the process
process[pid] = WAITING
status[pid] = "RESTART"
end
else # full load completed or failed, skip project
end
end
ensure
CSV.open(options['output'], 'w') do |outputfile|
input.each do |pid|
outputfile << [pid, process[pid], status[pid]]
end
end
end
status = {}
CSV.foreach(options['output'], :headers => false, :return_headers => false) do |row|
key = row[2]
if status.has_key?(key)
status[key] += 1
else
status[key] = 1
end
end
status.each do |s|
puts "#{s}"
end
puts "Downloading started at:"
started.sort_by(&:last).each do |p|
puts "#{p[0]} #{p[1]}"
end
end
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment