Soulmate Loader

soulmate.rb

require 'soulmate'

module Soulmate
  class Loader < Base
    def load_without_reset(items)
      items.each_with_index do |item, i|
        puts item["id"]
        add(item, :skip_duplicate_check => true)
      end
    end

    def reset
      # delete the sorted sets for this type
      phrases = Soulmate.redis.smembers(base)
      Soulmate.redis.pipelined do
        phrases.each do |p|
          Soulmate.redis.del("#{base}:#{p}")
        end
        Soulmate.redis.del(base)
      end

      # Redis can continue serving cached requests for this type while the reload is
      # occuring. Some requests may be cached incorrectly as empty set (for requests
      # which come in after the above delete, but before the loading completes). But
      # everything will work itself out as soon as the cache expires again.

      # delete the data stored for this type
      Soulmate.redis.del(database)
    end
  end
end


#files = ["./xaa", "./xab", "./xac", "./xad", "./xae", "./xaf", "./xag", "./xah", "./xai", "./xaj", "./xak"]
#files = ["./laa", "./xae", "./xaf", "./xag", "./xah", "./xai", "./xaj", "./xak"]
#files = ["./lab", "./xag", "./xah", "./xai", "./xaj", "./xak"]
files = ["./xai", "./xaj", "./xak"]
type = "hacker"
#loader = Soulmate::Loader.new(type)
#loader.reset

files.each do |filename|
  puts "Loading items of type #{type} from file #{filename}..."
  file = File.open(filename, "r")
  items = file.read.split("\n").map{|l| MultiJson.decode(l) }
  loader = Soulmate::Loader.new(type)
  loaded = loader.load_without_reset(items)
  puts "Loaded a total of #{loaded.size} items"
  file.close
end

puts "loading done"