-
-
Save korczis/a23a0d73bc6e3197aebe to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
source 'https://rubygems.org' | |
gem 'gooddata' | |
gem 'json' | |
gem 'pmap' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env ruby | |
require 'rubygems' | |
require 'gooddata' | |
require 'pp' | |
require 'json' | |
require 'csv' | |
require 'pmap' | |
require 'set' | |
require 'optparse' | |
require 'yaml' | |
DONE = 'SYNCHRONIZED' | |
WAITING = 'Another process is already running' | |
UNKNOWN = 'Another process' | |
TRYING = 'Trying to start' | |
ERROR = 'ERROR' | |
def read_input_csv_array(path) | |
res = [] | |
CSV.foreach(path, :headers => false, :return_headers => false) do |row| | |
res << row[0] | |
end | |
return res | |
end | |
def read_input_csv_hash(path) | |
res = {} | |
p = {} | |
CSV.foreach(path, :headers => false, :return_headers => false) do |row| | |
res[row[0]] = row[1] | |
p[row[0]] = row[2] | |
end | |
return res, p | |
end | |
def main | |
GoodData.logger = GoodData::NilLogger.new | |
begin | |
options = YAML.load_file('zd_config.yaml') | |
rescue | |
options = {} | |
end | |
OptionParser.new do |opts| | |
opts.banner = "Usage: ruby zd_massive_full_load.rb [options]" | |
opts.on('-u', '--username EMAIL', 'Username') { |v| options['username'] = v } | |
opts.on('-p', '--password PASSWORD', 'Password') { |v| options['password'] = v } | |
opts.on('-i', '--input FILE', 'Input File') { |v| options['input'] = v } | |
opts.on('-o', '--output FILE', 'Output File') { |v| options['output'] = v } | |
end.parse! | |
client = GoodData.connect(options['username'], options['password'], {:webdav_server => "https://na1-di.gooddata.com", :server => "https://analytics.zendesk.com"}) | |
puts "Logged in as #{options['username']}." | |
input = read_input_csv_array(options['input']) | |
process, status = read_input_csv_hash(options['output']) | |
output = [] | |
started = {} | |
begin | |
input.each do |pid| | |
if not process.has_key?(pid) or process[pid] == WAITING # full load was not started yet | |
begin | |
response = client.post("/gdc/projects/#{pid}/connectors/zendesk4/integration/processes", '{"process": {"incremental": false}}') | |
p = response['uri'] | |
process[pid] = p | |
status[pid] = TRYING | |
rescue RestClient::BadRequest => e # 400 Bad Request: Another process is running, cannot start full load | |
process[pid] = WAITING | |
response = client.get("/gdc/projects/#{pid}/connectors/zendesk4/integration") | |
begin | |
status[pid] = "#{UNKNOWN}: #{response['integration']['runningProcess']['status']['code']}" | |
rescue | |
status[pid] = UNKNOWN | |
end | |
rescue RestClient::Conflict => e | |
process[pid] = WAITING | |
status[pid] = "CONFLICT" | |
end | |
elsif status[pid] != DONE and status[pid] != ERROR # full load running, update status | |
begin | |
response = client.get(process[pid]) | |
s = response["process"]["status"]["code"] | |
status[pid] = s | |
started[pid] = response["process"]["started"] | |
rescue # status unknown, have to restart the process | |
process[pid] = WAITING | |
status[pid] = "RESTART" | |
end | |
else # full load completed or failed, skip project | |
end | |
end | |
ensure | |
CSV.open(options['output'], 'w') do |outputfile| | |
input.each do |pid| | |
outputfile << [pid, process[pid], status[pid]] | |
end | |
end | |
end | |
status = {} | |
CSV.foreach(options['output'], :headers => false, :return_headers => false) do |row| | |
key = row[2] | |
if status.has_key?(key) | |
status[key] += 1 | |
else | |
status[key] = 1 | |
end | |
end | |
status.each do |s| | |
puts "#{s}" | |
end | |
puts "Downloading started at:" | |
started.sort_by(&:last).each do |p| | |
puts "#{p[0]} #{p[1]}" | |
end | |
end | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment