Skip to content

Instantly share code, notes, and snippets.

@oshanz
Created January 30, 2020 08:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save oshanz/c18aa2a3a811519ae81ccebd39a0f440 to your computer and use it in GitHub Desktop.
Save oshanz/c18aa2a3a811519ae81ccebd39a0f440 to your computer and use it in GitHub Desktop.
Typhoeus parellel bug
module SisExports
class AssessmentsDataSource
def call(assessment_ids = [], fields = [], references = false, sis_token = nil)
hydra = Typhoeus::Hydra.new
batch_size = 100
master_key = fetch_master_key(assessment_ids, sis_token, fields, references)
requests = assessment_ids.each_slice(batch_size).map do |ids|
request = Typhoeus::Request.new(
assessments_url,
method: :post,
body: {
fields: fields,
references: references,
filter: {
ids: ids.join(',')
}
}.to_json,
params: { key: sis_token, masterKey: master_key },
headers: headers,
timeout: 1000
)
hydra.queue(request)
request
end
hydra.run
data = []
responses = requests.map do |r|
if r.response.success?
binding.pry
JSON.parse(r.response.body)['assessments']
else
raise r.response.body
end
end
binding.pry
puts "assessment count: #{assessment_ids.count}"
puts "data count: #{responses.count}"
responses
end
private
def headers
{
'Content-Type' => 'application/json',
token => token
}
end
def fetch_master_key(ids, sis_token, fields, references)
response = Typhoeus.post(
assessments_url,
headers: headers,
params: { key: sis_token },
body: {
fields: fields,
references: references,
filter: {
id: ids.first.to_s
}
}.to_json,
timeout: 1000
)
JSON.parse(response.body)['masterKey']
end
def assessments_url
#
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment