require 'redis' | |
require 'rss' | |
URLS = %W{https://news.ycombinator.com/rss | |
http://www.discoverdev.io/rss.xml} | |
redis = Redis.new | |
URLS.each { |url| redis.rpush "feeds_to_fetch", url } | |
loop do | |
queue, payload = redis.blpop "feeds_to_fetch", "entries_needing_processing", 0 | |
if queue == "feeds_to_fetch" | |
# Fetch feeds | |
feed_url = payload | |
puts "fetching feed: #{feed_url}" | |
content = open(feed_url) { |s| s.read } | |
rss = RSS::Parser.parse content, false | |
rss.items.each do |entry| | |
redis.rpush "entries_needing_processing", Marshal.dump([feed_url,entry]) | |
end | |
redis.rpush "feeds_to_fetch", feed_url | |
else | |
# Process entries | |
entry = Marshal.load payload | |
puts "processing entry: #{entry.url}" | |
entry_id = redis.incr "entries_processed" | |
redis.hmset "entry|#{entry_id}", "url", entry.url, "title", entry.title, | |
"published", entry.published, "description", entry.summary | |
redis.sadd "entry_ids", entry_id | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment