Scraping trains and stations list from indiantrains.org

scrape-indiantrains.org.rb

require 'net/http'
require 'uri'

# Fetch list of all stations with their codes
File.open("all_stations.csv","w") { |f|
(?A..?Z).to_a.collect(&:chr).each { |c|
  puts "Starting #{c}"
  sleep(3) #wait for 3 seconds
  uri = URI.parse("http://www.indiantrains.org/station-list/?navigate=" << c)
  response = Net::HTTP.get_response(uri)
  matches = response.body.scan(/http:\/\/www\.indiantrains\.org\/station-details\/\?code=([A-Z]+)&name=([A-Z+]+)"/).uniq
  matches.each { |m|
    name = m[1].gsub(/\+/,' ')
    f.puts "\"#{m.first}\",\"#{name}\""
  } #matches.each
} #chars.each
} #File.open

# Fetch list of all trains
File.open("all_trains.csv","w") { |f|
(?A..?Z).to_a.collect(&:chr).each { |c|
  puts "Starting #{c}"
  sleep(3)
  uri = URI.parse("http://www.indiantrains.org/train-list/?navigate=" << c)
  response = Net::HTTP.get_response(uri)
  matches = response.body.scan(/http:\/\/www\.indiantrains\.org\/train-details\/\?number=([1-9a-zA-Z]+)&name=([0-9a-zA-Z+()]+)"/).uniq
  matches.each { |m|
    name = m[1].gsub(/\+/,' ')
    f.puts "\"#{m.first}\",\"#{name}\""
  } #matches.each
} #chars.each
} #File.open