アーカイブのurl全部取得してdbに書き込む

collect_url.rb

# -*- encoding: utf-8 -*-

require 'nokogiri'
require "open-uri"
require 'sequel'

def insert(dbname, urls)
  db = Sequel.connect("sqlite://#{dbname}")
  urls.each do |url|
    db[:url].insert(value: url)
  end
end

def get_archive(url)
  ret = []
  charset = nil
  html = open(url) do |f|
    charset = f.charset
    f.read
  end
  doc = Nokogiri::HTML.parse(html, nil, charset)
  doc.xpath('//h1[@class="article-title"]/a').each do |v|
    ret << v.attribute('href').value
  end
  ret
end

for l in 59..100 do
  insert('url.db', get_archive("http://ankake.blog.jp/?p=#{l}"))
end

#insert('url.db', 'http://ankake.blog.jp/archives/1008878299.html')
#puts get_archive('http://ankake.blog.jp/?p=30')