tao_buoy_scraper.rb

require 'mechanize'
require 'open-uri'

mechanize = Mechanize.new

base_url = 'http://tao.ndbc.noaa.gov/tao/data_download/'

all_buoys = 'TR8S95E.TR5S95E.TR2S90E.T0N85W.T8S95W.T5S95W.T2S95W.T0N95W.T5N95W.T2N95W.T10N95W.T8N95W.T12N95W.T8N110W.T5N110W.T2N110W.T1N110W.T8S110W.T5S110W.T2S110W.T1S110W.T0N108W.T0N111W.T0N110W.T8S125W.T2S125W.T0N125W.T5S125W.T2N125W.T5N125W.T8N125W.T7N132W.T5S140W.T2N140W.T0N140W.T2S140W.T5N140W.T9N140W.T7N147W.T8N155W.T5N155W.T2N155W.T2N157W.T0N155W.T1N153W.T0N152W.T1S153W.T8S155W.T5S155W.T2S155W.T8S170W.T2S170W.T0N170W.T5S170W.T2N170W.T5N170W.T8N170W.T0N176W.T8S180W.T5S180W.T5N180W.T2N180W.T2S180W.T0N180W.T8N180W.T0N170E.T1S167E.T8S165E.T5S165E.T2S165E.T0N165E.T2N165E.T5N165E.T8N168E.T8N167E.T8N165E.T0N161E.TR2S156E.TR5S156E.TR0N156E.TR2N156E.TR5N156E.TR8N156E.TR5N147E.TR0N147E.TR2N147E.TR0N138E.TR2N138E.TR5N137E.TR8N137E.TR2N130E.TR5N130E.TR8N130E'.split('.')

def get_file(url, file_name) 
  File.open("files/" + file_name,  "wb") do |saved_file|
    # the following "open" is provided by open-uri
    open(url, "rb") do |read_file|
      saved_file.write(read_file.read)
    end

    puts "Saved #{file_name}"
  end
end 

all_buoys[0, 3].each do |buoy|
  buoy_url = "http://tao.ndbc.noaa.gov/tao/data_download/process_results.php?type=sst&reso=daily&year1=1977&mon1=01&day1=01&year2=2017&mon2=02&day2=04&format=ascii&compression=none&stations=#{buoy}&historical=na"

  puts "Fetching page for #{buoy}"
  page = mechanize.get(buoy_url)
  puts "Got page #{page}"

  links = page.css('#content table td a')
  puts "Got links #{links}"

  links.each do |link|
    download_link = link['href']
    file_name = link.text
    full_link = base_url + download_link
    puts "Trying file #{file_name} -- got link address #{full_link} "
    get_file(full_link, file_name)
  end
end
puts "Done!"