tommetge/wot-pov-scraper.rb

## wot-pov-scraper.rb
#!/usr/bin/env ruby

require 'rubygems'
require 'mechanize'

# Utility methods
def merge_povs(first_pov, second_pov)
  second_pov.each do |k, v|
    sum = [first_pov[k].to_i, second_pov[k].to_i].inject(:+)
    first_pov[k] = sum
  end

  return first_pov
end

# Scraper methods
def get_chapter_povs(agent, chapter_url)
  puts "  Fetching #{chapter_url}..."
  chapter = agent.get(chapter_url)
  povs = chapter.search("p").text.scan(/[a-zA-Z0-9].* POV/)
  chapter_povs = {}
  povs.each do |pov|
    if chapter_povs[pov]
      chapter_povs[pov] = chapter_povs[pov] + 1
    else
      chapter_povs[pov] = 1
    end
  end

  return chapter_povs
end

def get_book_pov(agent, starting_page, book_url)
  puts "Fetching #{book_url}..."
  book_povs = {}
  book = agent.get(starting_page + "/" + book_url)

  # Scrape chapter URLs
  chapters = book.search("ol li a").map do |ch_src|
    ch_src.attributes["href"].value
  end
  # Add prologues and friends
  book.search("ul li a").each do |extra|
    chapters << extra.attributes["href"].value
  end
  chapters.each do |chapter_url|
      povs = get_chapter_povs(agent, chapter_url) rescue {}
      merge_povs(book_povs, povs)
  end

  return book_povs
end

# The real business: scraping books for POVs
def main
  agent = Mechanize.new

  starting_page = "http://encyclopaedia-wot.org"
  page = agent.get(starting_page)

  books = page.search('ol li a').map do |book_src|
    book_src.attributes["href"].value
  end

  final_povs = {}
  books.each do |book|
    povs = get_book_pov(agent, starting_page, book)
    pp povs
    merge_povs(final_povs, povs)
  end

  pp final_povs
end

main
	#!/usr/bin/env ruby

	require 'rubygems'
	require 'mechanize'

	# Utility methods
	def merge_povs(first_pov, second_pov)
	second_pov.each do \|k, v\|
	sum = [first_pov[k].to_i, second_pov[k].to_i].inject(:+)
	first_pov[k] = sum
	end

	return first_pov
	end

	# Scraper methods
	def get_chapter_povs(agent, chapter_url)
	puts " Fetching #{chapter_url}..."
	chapter = agent.get(chapter_url)
	povs = chapter.search("p").text.scan(/[a-zA-Z0-9].* POV/)
	chapter_povs = {}
	povs.each do \|pov\|
	if chapter_povs[pov]
	chapter_povs[pov] = chapter_povs[pov] + 1
	else
	chapter_povs[pov] = 1
	end
	end

	return chapter_povs
	end

	def get_book_pov(agent, starting_page, book_url)
	puts "Fetching #{book_url}..."
	book_povs = {}
	book = agent.get(starting_page + "/" + book_url)

	# Scrape chapter URLs
	chapters = book.search("ol li a").map do \|ch_src\|
	ch_src.attributes["href"].value
	end
	# Add prologues and friends
	book.search("ul li a").each do \|extra\|
	chapters << extra.attributes["href"].value
	end
	chapters.each do \|chapter_url\|
	povs = get_chapter_povs(agent, chapter_url) rescue {}
	merge_povs(book_povs, povs)
	end

	return book_povs
	end

	# The real business: scraping books for POVs
	def main
	agent = Mechanize.new

	starting_page = "http://encyclopaedia-wot.org"
	page = agent.get(starting_page)

	books = page.search('ol li a').map do \|book_src\|
	book_src.attributes["href"].value
	end

	final_povs = {}
	books.each do \|book\|
	povs = get_book_pov(agent, starting_page, book)
	pp povs
	merge_povs(final_povs, povs)
	end

	pp final_povs
	end

	main