used to download pdfs from the OSTI.GOV information bridge

download_urls_from_osti

require 'rubygems'
require 'mechanize'
require 'progressbar'

docs = File.readlines('URLS.txt')
pb = ProgressBar.new('downloading', docs.length)

docs.each do |line|
  url = line.strip
  if !File.exist?(File.basename(url))
    agent = Mechanize.new
    agent.get(url) # get a session cookie
    f = File.new(File.basename(url),'wb')
    f << agent.get(url).body # download the file
    sleep rand 5
    f.close
  end
  pb.inc
end

requires an adjacent file named 'URLS.txt' containing lines of addresses like:

http://www.osti.gov/bridge/servlets/purl/972223-KyjlCP/972223.pdf
http://www.osti.gov/bridge/servlets/purl/5486534-BMf4MW/5486534.pdf
http://www.osti.gov/bridge/servlets/purl/6489792-nii8Km/6489792.pdf