#download all files listed in an index page #ruby #nokogiri

download_index.rb

#!/usr/bin/env ruby

# Given an index page listing files,
# download all files listed.
#
# License: 0BSD

require 'nokogiri'
require 'open-uri'

def download_all(url)
  html = Nokogiri.parse(open(url).read)
  links = html.css('a').map { |l| l['href'] }
  rls = links.select { |l| l.match(/\.[a-z0-9]{3,4}$/) }
  dls = rls.map { |l| url + l }
  dls.each { |l| system('wget', '-c', l) }
end

if __FILE__ == $0
  if ARGV.length == 0
    puts 'Usage: download_index URL ..'
  elsif ARGV.length == 1
    download_all(ARGV[0])
  else
    ARGV.each do |url|
    directory_name = url.match(/\/[^\/]+\/$/).to_s.gsub('/', '')
    FileUtils.mkdir_p directory_name
    FileUtils.cd directory_name
    download_all(url)
    FileUtils.cd '..'
end