fashion press cralwer

fashion_press.rb

#!/bin/env ruby
# encoding: utf-8

require 'net/http'
require 'nokogiri'

uri = URI("http://www.fashion-press.net/brands/en")
doc = Nokogiri::HTML(Net::HTTP.get(uri))

results = []
doc.search("#brandlist > table > tr > td > ul > li > a").each { |a|
  en_name = a.text
  body_url = a.attributes["href"].value
  uri.path = body_url

  resp = Net::HTTP.get_response(uri)
  if resp.code.to_i == 200
    inner_doc = Nokogiri::HTML(resp.body)
    kana = inner_doc.search(".title_sub").text
    puts [en_name, kana]
    results << [en_name, kana].join("\t")
    sleep 0.2
  else
    STDERR.puts "failed fetch [#{resp.code}] #{uri} ignored"
    next
  end
}

File.write("fashion_brand_master.tsv", results.join("\n"))