Curl url downloader as an alternative to curb

url_loader.rb

# Helper class to save urls to a file
class UrlLoader
  # Download a url and optionally specify a path to save it to a file.
  def self.get(url, path=nil)
    # set curl to timeout after 30 minutes for any calls
    cmd = "curl --fail --location --silent --show-error --max-redirs 10 --connect-timeout 60 --max-time 1800"
    cmd += " --output #{path}" unless path.nil?
    cmd += " '#{url}'"
    rsp = `#{cmd}`
    raise "Received non-success response code for url: '#{url}'" if (path.nil? ? rsp.empty? : (!File.exists?(path) || File.size(path) == 0))
    path.nil? ? rsp : File.new(path)
  end
end