once filter

once.rb

#!/usr/bin/env ruby
# -*- coding: utf-8 -*-

# For example:
#
#   $ ruby your-scraper.rb | ruby once.rb | tee -a new-list.txt
#   foo
#   bar
#   :
#   buzz
#
#   $ ruby your-scraper.rb | ruby once.rb | tee -a new-list.txt
#   new_foo
#
# ref. https://github.com/tily/cirrocumulus/blob/master/app.rb

# Gemfile memo
# gem 'redis', '~> 3.1'
# gem 'redis-pool', '~> 0.1.1'
# gem 'redis-namespace', '~> 1.5.1'

require 'redis'
require 'redis/pool'
require 'redis-namespace'

class Once
  def redis
    @redis ||= Redis::Namespace.new(:once, 
      redis: Redis::Pool.new(
        url: ENV['REDISTOGO_URL'] || 'redis://localhost:6379/1/'))
  end

  def known?(x)
    if redis.get(x)
      true
    else
      redis.set(x, true)
      redis.persist x
      # redis.expire(x, 60*60*24*365*100)
      false
    end
  end
end

#binding.pry

if $PROGRAM_NAME == __FILE__
  once = Once.new
  list = ARGF.each_line.map { |x| x.chomp }.
    delete_if { |x| once.known? x }
  puts list
end