How to Build a Fast News Feed in Redis and Rails

redis_helper.rb

module RedisHelper
  # decode Redis value back to Ruby object
  def self.decode(json)
    self.new(ActiveSupport::JSON.decode(json)["#{self.name.downcase}"])
  end

  # encode Ruby object for Redis
  def encoded
    self.updated_at = nil
    self.to_json
  end

  # helpers to generate Redis keys
  def timestamp
    "#{self.created_at.to_i}"
  end

  def key(str, uid=self.id)
    "#{str}:#{uid}"
  end

  def ukey(str, uid=self.user_id) #for keys needing user_id
    "#{str}:#{uid}"
  end

  def id_s
    id.to_s
  end
end

status.rb

class User 

  FEED_COUNT=30

  # get latest feed using reverse range lookup of sorted set
  # then decode raw JSON back into Ruby objects
  def feed(obj=true, count=FEED_COUNT)
    results=$redis.zrevrange key(:feed), 0, count
    if obj && results.size > 0
      results.collect {|r| Status.decode(r)}
    else
      results
    end
  end

  # get older statuses by using reverse range by score lookup
  def ofeed(max, obj=true, id=self.id_s, limit=FEED_COUNT, scores=false)
    results=$redis.zrevrangebyscore(key(:feed), "(#{max}", "-inf", :limit => [0, limit], :with_scores => scores)
    if obj && results.size > 0
      results.collect {|r| Status.decode(r)}
    else
      results
    end     
  end

end


class Status

  # push status to a specific feed
    def push(id, location="feed")
      $redis.zadd key(location, id), timestamp, encoded
    end

  # push to followers (assumes an array of follower ids)
  def push_to_followers
    @follower_ids.each do |follower_id|
      push(follower_id)
    end
  end

end

# also in user.rb: 

  # since most sorted set commands are a variation on O(log(N)) 
  # where N is the size of the set, in makes sense to trim the
  # feed when it gets beyond a certain length

  FEED_LENGTH=240

  # there may be a more efficient way to do this
  # but I check the length of the set
  # then I get the score of the last value I want to keep
  # then remove all keys with a lower score
  def trim_feed(id=self.id_s, location="feed", indx=FEED_LENGTH)
    k = key(:feed)
    if ($redis.zcard k) >= indx
      n = indx - 1
      if (r = $redis.zrevrange k, n, n, :with_scores => true)
        $redis.zremrangebyscore k, "-inf", "(#{r.last}"
      end
    end
  end

It's safer/faster to put just a pointer to the post in the feed, eg. the posts id field, and just make multiple database requests. That way you don't fill up the cache, as a redis database can only be as big as it's host memory.

Because if a post was getting pushed out to 10,000 people, and you had 3kb of JSON in the post, you'd have 29.3mb of data in the cache!