Skip to content

Instantly share code, notes, and snippets.

@havenwood

havenwood/fetch.rb

Last active Dec 24, 2020
Embed
What would you like to do?
My little script to download Ruby source files in parallel with Async
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'async'
require 'async/barrier'
require 'async/http/client'
require 'async/http/endpoint'
require 'async/logger'
require 'optionparser'
options = ARGV.options do |command|
command.banner = "Usage: #{command.program_name} ruby_version"
command.version = '0.0.1'
command
end
options.freeze.permute!
ruby_version = ARGV.shift
raise ArgumentError, "missing argument: ruby_version\n#{options.banner}" unless ruby_version
host = 'https://cache.ruby-lang.org'
major, minor, *_rest = ruby_version.split('.')
dir = File.join('pub', 'ruby', "#{major}.#{minor}")
basename = "ruby-#{ruby_version}"
extensions = %w[tar.gz tar.xz zip].freeze
Async do
endpoint = Async::HTTP::Endpoint.parse(host)
client = Async::HTTP::Client.new(endpoint)
barrier = Async::Barrier.new
extensions.each do |extension|
file = "#{basename}.#{extension}"
path = File.join(dir, file)
url = File.join(host, path)
barrier.async do |task|
Async.logger.info "Requesting #{url} ..."
response = client.get(path)
if response.success?
bytes = response.body.length.digits.each_slice(3).map(&:join).join(',').reverse
Async.logger.info "Downloading #{file}, #{bytes} bytes ..."
response.save(file)
Async.logger.info "Downloaded #{file}, #{bytes} bytes saved ..."
else
if response.valid?
Async.logger.error "Downloading #{file} failed with a #{response.status} error status ..."
else
Async.logger.error "Downloading #{file} failed with invalid response ..."
end
task.stop
end
end
end
barrier.wait
ensure
client&.close
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment