scrap.rb

require 'hpricot'
require 'open-uri'
require 'date'

class Post
    attr_accessor :author, :title, :date, :tags, :comments

    def initialize url
        doc = Hpricot(open(url))
        @author = doc.search("//author/name").fetch(1).inner_text
        @title = doc.search("//title/").fetch(1).inner_text
        @date = DateTime.parse(doc.at("//published").inner_text)
        @comments = doc.inner_html.match(/title="(\d+) Comments\"/)[1].to_i
    end

    def printout
        [@title,
            @author,
            @date.httpdate,
            #@tags.join(', '),
            @comments].join("\n")
    end

end

File.open("model") do |f|
    posts = Marshal.load(f.read)
    File.open("new_output", "w+") do |file|
        posts.each do |post|
            file.puts post.title
            file.puts post.author
            file.puts post.comments
            file.puts post.date.httpdate
            file.puts Date::DAYNAMES[post.date.wday]
            file.puts post.date.mday
            file.puts Date::MONTHNAMES[post.date.month]
            file.puts post.date.year
            file.puts "----"
        end
    end
end


#posts = []

#i = 1

#File.open("output","w+") do |f|
#    while i <= 2168 do
#        begin
#            posts.push Post.new("http://www.fivethirtyeight.com/feeds/posts/default?start-index=#{i}&max-results=#{i}")
#            f.puts posts.last.printout
#            f.puts "----"
#        rescue RuntimeError => e
#            puts "Error: #{e}"
#        else
#            printf("%4.2f%\n", i / 2168.0 * 100)
#            i += 1
#        end
#    end
#end
#
#File.open("model","w+") do |f|
#    f.puts Marshal.dump(posts)
#end