Skip to content

Instantly share code, notes, and snippets.

@rbnpercy rbnpercy/rss_redis.rb
Last active Aug 14, 2019

Embed
What would you like to do?
require 'redis'
require 'rss'
URLS = %W{https://news.ycombinator.com/rss
http://www.discoverdev.io/rss.xml}
redis = Redis.new
URLS.each { |url| redis.rpush "feeds_to_fetch", url }
loop do
queue, payload = redis.blpop "feeds_to_fetch", "entries_needing_processing", 0
if queue == "feeds_to_fetch"
# Fetch feeds
feed_url = payload
puts "fetching feed: #{feed_url}"
content = open(feed_url) { |s| s.read }
rss = RSS::Parser.parse content, false
rss.items.each do |entry|
redis.rpush "entries_needing_processing", Marshal.dump([feed_url,entry])
end
redis.rpush "feeds_to_fetch", feed_url
else
# Process entries
entry = Marshal.load payload
puts "processing entry: #{entry.url}"
entry_id = redis.incr "entries_processed"
redis.hmset "entry|#{entry_id}", "url", entry.url, "title", entry.title,
"published", entry.published, "description", entry.summary
redis.sadd "entry_ids", entry_id
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.