Archive a FriendFeed feed in MongoDB

friendfeed2mongodb.rb

namespace :db do
  require "mongo"
  require "json/pure"
  require "open-uri"

  feed    = ENV['feed']
  db      = Mongo::Connection.new.db('friendfeed')
  col     = db.collection('entries')

  desc "Seed database with feed entries"
  task :seed do
    0.step(9900, 100) do |n|
      j = JSON.parse(open("http://friendfeed-api.com/v2/feed/#{feed}?start=#{n}&num=100").read)
      break if j['entries'].count == 0
      entries = j['entries']
      j.delete('entries')
      j['updated_at'] = Time.now
      entries.each do |entry|
        entry['_id'] = "#{j['sup_id']}/#{entry['id']}"
        entry.delete('id')
        entry['feed'] = j
        col.save(entry)
      end
      puts "Processed entries #{n} - #{n + 99}"
      sleep(3)
    end
    puts "Done: database contains #{col.count} entries."
  end

end

1. Re-written as a rake task; save as "Rakefile" and run as "rake db:seed feed=FEED_ID".
2. entry ID alone not sufficient as unique key (may appear in several feeds); so prepended feed sup_id.

Changed step back to 9900; don't think anything above this returns more results.

Added a sleep() to this version of code.