crawler.rb

require 'date'

require 'mechanize'
require 'nokogiri'
require 'csv'

agent = Mechanize.new

start_date = DateTime.new(2010, 2, 1)
end_date = DateTime.new(2010, 2, -1)

csv = CSV.open("/tmp/output.csv", "w")

until start_date.year == 2016 && start_date.month == 11 do
  puts start_date.strftime("%Y/%m/%d")

  options = {
    ymdF: start_date.strftime("%Y/%m/%d"),
    hmsF: "00:00",
    ymdT: end_date.strftime("%Y/%m/%d"),
    hmsT: "23:59",
    MaxI: "I1",
    MinM: "F00",
    MaxM: "T95",
    DepF: "0",
    DepT: "999",
    "EpiN[]" => "999",
    "Pref[]" => "35",
    "City[]" => "99999",
    "Obs[]" => "9999999",
    Int: "1",
    Sort: "S0",
    Count: "",
    Comp: "C1",
    DetailFlg: "1"
  }
  page = agent.post("http://www.data.jma.go.jp/svd/eqdb/data/shindo/index.php", options)
  doc = Nokogiri::HTML(page.body)

  doc.xpath("//table[@id='CountList']//tr").drop(1).each do |row|
    tarray = []
    row.xpath('th|td').each do |cell|
      break if cell.text == "合計"
      tarray << cell.text
    end
    csv << tarray if !tarray.empty?
  end

  start_date = start_date >> 1
  end_date = DateTime.new(start_date.year, start_date.month, -1)
  sleep 0.5
end

csv.close