mass downloadin

download.rb

require 'oga'
require 'open-uri'
require 'uri'
require 'rest-client'
require 'pry'
require 'thread'

$log_lock = Mutex.new
def get_urls
  page_url = ARGV[0]
  page = Oga.parse_xml RestClient.get(page_url)

  links = page.xpath("//a").map { |l| l.get "href" }
  links.shift
  linkurls = links.map {|b| URI.encode(page_url+b)}

  File.open("urls","w"){ |f| f << linkurls}
end

def download_files
  get_urls unless File.exists? "urls"
  urls = File.open("urls") {|f|eval(f.read)}
  threads = []
  urls.each { |url|
    threads << create_download_thread(url)
  }

  threads.each {|t| t.join}
end

def create_download_thread download
  Thread.new {
    $log_lock.synchronize { puts "downloading #{download}..." }
    RestClient.get(download){ |response, request, result,&block|
      name = URI.decode File.basename(download)
      $log_lock.synchronize {  puts "downloaded #{download} ! "}
      File.open(name, "w") { |f| f << response}
    }
  }
end

download_files