Fetch the Queen's new-years speeches into a SQLite3 database... WHY WOULDN'T YOU?

Gemfile

source :gemcutter

gem 'hpricot'
gem 'dm-core'
gem 'dm-sqlite-adapter'
gem 'dm-migrations'

INSTRUCTIONS

$ gem install bundler
$ bundle install
$ ruby queenish.rb

queenish.rb

# encoding: utf-8

# Fetch the Queen's new-years speeches into a SQLite3 database... WHY WOULDN'T YOU?

require "rubygems"
require "bundler"
Bundler.require(:default)
require 'open-uri'

class Speech
  include DataMapper::Resource

  property :id, Serial
  property :year, String
  property :body, Text
end

FileUtils.rm_rf("data.sqlite3")
DataMapper.setup(:default,
  "sqlite3:///#{Dir.pwd}/data.sqlite3")
DataMapper.auto_migrate!

def fetch_and_convert(url)
  open(url, "r:iso-8859-1").read.encode("utf-8")
end

index = Hpricot(fetch_and_convert("http://kongehuset.dk/publish.php?dogtag=k_dk_aktuelt_taler"))
links = (index / "div#content a").select { |a| a.html =~ /tale/i }

links.first(1).each do |tale|
  h = Hpricot(fetch_and_convert(tale.get_attribute("href")))
  year, body = [tale.html,
    (h / "div#content").to_html.gsub!(/(<[^>]*>)/,"")]
  puts body
  Speech.new(:year => year, :body => body).save
end