filter_grep benchmark

grep-benchmark.rb

require_relative "../lib/fluent/load"
require_relative "../lib/fluent/plugin/filter_grep"
require_relative "../lib/fluent/plugin/filter_grep_new1"
require_relative "../lib/fluent/plugin/filter_grep_new2"
require_relative "../lib/fluent/test"
require_relative "../lib/fluent/test/driver/filter"

require "benchmark"
require "benchmark/ips"

def create_driver(conf = '', mode = 1)
  case mode
  when 1
    Fluent::Test::Driver::Filter.new(Fluent::Plugin::GrepFilter).configure(conf)
  when 2
    Fluent::Test::Driver::Filter.new(Fluent::Plugin::Grep2Filter).configure(conf)
  when 3
    Fluent::Test::Driver::Filter.new(Fluent::Plugin::Grep3Filter).configure(conf)
  end
end

@time = Fluent::EventTime.now

def filter(d, messages)
  d.run do
    messages.each do |message|
      d.feed("filter.test", time, message)
    end
  end
  d.filtered_records
end

Fluent::Test.setup

conf = %[
   <and>
     <regexp>
       key level
       pattern ^ERROR|WARN$
     </regexp>
     <regexp>
       key method
       pattern ^GET|POST$
     </regexp>
   </and>

   <or>
     <regexp>
       key level
       pattern ^ERROR|WARN$
     </regexp>
     <regexp>
       key method
       pattern ^PUT$
     </regexp>
   </or>

   <or>
     <exclude>
       key level
       pattern ^WARN$
     </exclude>
     <exclude>
       key method
       pattern ^GET$
     </exclude>
   </or>

   <and>
     <exclude>
       key level
       pattern ^INFO$
     </exclude>
     <exclude>
       key method
       pattern ^DELETE$
     </exclude>
   </and>

]

messages = [
  { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" },
  { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" },
  { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" },
  { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" },
  { "time" => "2013/01/13T07:02:13.232645", "level" => "ERROR", "method" => "PUT", "path" => "/edit" },
  { "time" => "2013/01/13T07:02:21.542145", "level" => "INFO", "method" => "PUT", "path" => "/edit" },
  { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "PUT", "path" => "/edit" },
  { "time" => "2013/01/13T07:02:21.542145", "level" => "INFO", "method" => "DELETE", "path" => "/article/1" },
  { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "DELETE", "path" => "/article/2" },
  { "time" => "2013/01/13T07:02:43.632145", "level" => "ERROR", "method" => "DELETE", "path" => "/article/3" },
]

Benchmark.ips do |x|
  x.time = 10
  x.report(:original) do
    d = create_driver(conf, 1)
    d.run do
      messages.each do |message|
        d.feed("filter.test", @time, message)
      end
    end
  end
  x.report("1940") do
    d = create_driver(conf, 2)
    d.run do
      messages.each do |message|
        d.feed("filter.test", @time, message)
      end
    end
  end
  x.report("1940+") do
    d = create_driver(conf, 3)
    d.run do
      messages.each do |message|
        d.feed("filter.test", @time, message)
      end
    end
  end

  x.compare!
end