Picky Realtime Indexing

app.rb

require 'bundler'
Bundler.require

BookIndex = Picky::Index.new :books do
  category "titel"
  category "autor"
  category "hrsg"
  category "signatur"
  category "jahr"
  category "kommentar"
  category "anmerkungen"
  category "auflage"
  category "baende"
  category "bearbeiter"
  category "bestelladresse"
  category "drehbuch"
  category "format"
  category "inhalt"
  category "sprache"
  category "literaturvorlage"
  category "nebensignatur"
  category "ort"
  category "platz"
  category "preis"
  category "publikationsart"
  category "reihe"
  category "seiten"
  category "stifter"
  category "verlag"
  category "standort"
  category "datensatz"
  category "aufnahmedatum"
  category "issn"
  category "isbn"
  category "invent"
  category "autor_japanisch"
  category "hrsg_japanisch"
  category "drehbuch_japanisch"
  category "reihe_japanisch"
  category "titel_japanisch"
  category "verlag_japanisch"
  category "literaturvorlage_japanisch"
  category "nacsis_japanisch"
  category "jid"
  category "nacsis_url"
  category "interne_notizen"
  category "created_at"
  category "updated_at"
  category "vormerken"
  category "altes_datum"

  indexing  removes_characters: /[^\p{Han}\p{Hiragana}\p{Katakana}a-zA-Z0-9\.\s]/u,
            substitutes_characters_with: Picky::CharacterSubstituters::WestEuropean.new,
            splits_text_on: /\s/u
end

class TestApp < Sinatra::Base
  extend Picky::Sinatra::IndexActions
end

require 'rack/test'

class TestRunner
  include Rack::Test::Methods
  def app
    @app ||= TestApp
  end

  def rand_data i
    h = {}
    BookIndex.categories.each do |c|
      h[c.name] = "Lorem ipsum dolor sit amet #{c.name} #{i}"
    end
    h[:id] = i
    MultiJson.encode(h)
  end

  def generate_data
    @data = []
    0.upto(100) do |i|
      @data << rand_data(i)
    end
  end

  def fill_index backend
    BookIndex.backend backend

    @data.each do |datum|
      put "/", {index: "books", data: datum}
    end
  end

  def run
    puts "\n"
    backends = [Picky::Backends::Memory.new, Picky::Backends::Redis.new({realtime: true})]
    backends.each do |backend|
      puts "Trying #{backend.class}"
      generate_data
      start_time = Time.now
      fill_index backend
      stop_time = Time.now
      puts "Time in seconds:"
      puts stop_time - start_time
    end
  end
end

TestRunner.new.run

Gemfile

gem 'picky'
gem 'redis'
gem 'sinatra', require: 'sinatra/base'
gem 'rack-test'
gem 'pry'

Output is:

Trying Picky::Backends::Memory
Time in seconds:
2.207154854
Trying Picky::Backends::Redis
Time in seconds:
354.592928943

Here is what I get:

~/temp/gists/4709109 $ ruby app.rb 

Trying Picky::Backends::Memory
Time in seconds:
0.143306
Trying Picky::Backends::Redis
Time in seconds:
0.329048

On a 2012 model MacBook Pro with:

• Redis server version 2.4.16 (00000000:0)
• ruby 1.9.3p194 (2012-04-20 revision 35410) [x86_64-darwin12.1.0]
• Picky 4.12.7

And the Gemfile.lock:

GEM
  specs:
    activesupport (3.2.11)
      i18n (~> 0.6)
      multi_json (~> 1.0)
    coderay (1.0.8)
    i18n (0.6.1)
    method_source (0.8.1)
    multi_json (1.5.0)
    picky (4.12.7)
      activesupport (>= 3.0)
      multi_json
      rack_fast_escape
      text
    pry (0.9.10)
      coderay (~> 1.0.5)
      method_source (~> 0.8)
      slop (~> 3.3.1)
    rack (1.4.4)
    rack-protection (1.3.2)
      rack
    rack-test (0.6.2)
      rack (>= 1.0)
    rack_fast_escape (2009.06.24)
      url_escape
    redis (3.0.2)
    sinatra (1.3.4)
      rack (~> 1.4)
      rack-protection (~> 1.3)
      tilt (~> 1.3, >= 1.3.3)
    slop (3.3.3)
    text (1.2.1)
    tilt (1.3.3)
    url_escape (2009.06.24)

PLATFORMS
  ruby

DEPENDENCIES
  picky
  pry
  rack-test
  redis
  sinatra

With Picky 4.12.9: much worse! I'll investigate.

With Ruby 4.12.9:

~/temp/gists/4709109 $ ruby app.rb 

Trying Picky::Backends::Memory
Time in seconds:
1.484084
Trying Picky::Backends::Redis
Time in seconds:
276.306228

4.12.8 is bad also, the change has been introduced in 4.12.8.

LOL.
Complete fail on my part – of course it is fast on 4.12.7: the server does not use PUT with that version, resulting in 404s. Please ignore all comments.