rss_redis.rb

require 'redis'
require 'rss'


URLS = %W{https://news.ycombinator.com/rss
          http://www.discoverdev.io/rss.xml}

redis = Redis.new

URLS.each { |url| redis.rpush "feeds_to_fetch", url }

loop do
  queue, payload = redis.blpop "feeds_to_fetch", "entries_needing_processing", 0
  if queue == "feeds_to_fetch"
    # Fetch feeds
    feed_url = payload
    puts "fetching feed: #{feed_url}"
    content = open(feed_url) { |s| s.read }
    rss = RSS::Parser.parse content, false
    rss.items.each do |entry|
      redis.rpush "entries_needing_processing", Marshal.dump([feed_url,entry])
    end
    redis.rpush "feeds_to_fetch", feed_url
  else
    # Process entries
    entry = Marshal.load payload
    puts "processing entry: #{entry.url}"
    entry_id = redis.incr "entries_processed"
    redis.hmset "entry|#{entry_id}", "url", entry.url, "title", entry.title, 
                "published", entry.published, "description", entry.summary
    redis.sadd "entry_ids", entry_id
  end
end