in_tail_asis とか in_tail の none parser で読み込んだ FortiGate の syslog を parse する。 FortiGate のログは項目が多いので keys か remove_keys で必要なものだけに絞る。 loop で gsub! するのは非常に重いので書き直した。

bench.rb

require 'benchmark'

lines = []
f = open(ARGV[0])
while line = f.gets
  lines.push line.chomp
  break
end
f.close

n = 10000
Benchmark.bm do |x|
  x.report("A: ") {
    (1..n).each {
      lines.each { |line|
        record = {}
        data = line.split(/\s+/, 5).pop
        while data != '' do
          data.gsub!(/^([a-zA-Z0-9_]+)=("[^"]*"|\S+)(:?\s|$)/, '')
          record[$1] = $2.gsub(/^"(.*)"$/, '\1')
        end
        #puts record
      }
    }
  }
  x.report("B: ") {
    (1..n).each {
      lines.each { |line|
        record = {}
        data = line.split(/\s+/, 5).pop
        data.gsub(/\G[a-zA-Z0-9_]+=(:?"[^"]*"|\S+)(:?\s|$)/) { |kv|
            (k, v) = kv.strip.split(/=/, 2)
            record[k] = v.gsub(/^"(.*)"$/, '\1')
        }
        #puts record
      }
    }
  }
end

gistfile1.txt

# ./bench.rb forti.1000 
       user     system      total        real
A:  15.190000   0.000000  15.190000 ( 15.193379)
B:   3.870000   0.000000   3.870000 (  3.871790)

out_fortigate_syslog_parser.rb

module Fluent
  class FortigateSyslogParseOutput < Output
    Fluent::Plugin.register_output('forti_log_parser', self)

    config_param :remove_prefix,   :string, :default => nil
    config_param :add_prefix,      :string, :default => nil
    config_param :message_key,     :string, :default => 'message'
    config_param :keys,            :string, :default => nil
    config_param :remove_keys,     :string, :default => nil

    def configure(conf)
      super

      if @remove_prefix
        @removed_prefix_string = @remove_prefix + '.'
        @removed_length = @removed_prefix_string.length
      end
      if @add_prefix
        @added_prefix_string = @add_prefix + '.'
      end

      if @keys
        if @remove_keys
          raise ConfigError, "forti_log_parser: 'keys' and 'remove_keys' parameters are exclusive"
        end
        @keys = Hash[@keys.split(',').map {|x| [x, 1] }]
      end
      if @remove_keys
        @remove_keys = Hash[@remove_keys.split(',').map {|x| [x, 1] }]
      end
    end

    def emit(tag, es, chain)
      _tag = tag.clone

      if @remove_prefix and
        ((tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) || tag == @remove_prefix)
        tag = tag[@removed_length..-1] || ''
      end

      if @add_prefix
        tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
      end

      es.each do |time, record|
        time, record = parse(record)
        Engine.emit(tag, time, record)
      end

      chain.next
    end

    def parse(record)
      message = record[@message_key]
      record.delete(@message_key)
      data = message.split(/\s+/, 5).pop
      data.gsub(/\G[a-zA-Z0-9_]+=(:?"[^"]*"|\S+)(:?\s|$)/) { |kv|
        (k, v) = kv.strip.split(/=/, 2)
        if (k == 'date' or k == 'time' or
           (@keys and @keys.has_key?(k)) or
           (@remove_keys and not @remove_keys.has_key?(k)) or
           (!@keys and !@remove_keys))
          record[k] = v.gsub(/^"(.*)"$/, '\1')
        end
      }

      time = Time.strptime(record["date"] + " " + record["time"], '%Y-%m-%d %H:%M:%S').to_i

      record.delete("date")
      record.delete("time")

      [ time, record ]
    end
  end
end

out_fortigate_syslog_parser_orig.rb

module Fluent
  class FortigateSyslogParseOutput < Output
    Fluent::Plugin.register_output('forti_log_parser', self)

    config_param :remove_prefix,   :string, :default => nil
    config_param :add_prefix,      :string, :default => nil
    config_param :message_key,     :string, :default => 'message'
    config_param :keys,            :string, :default => nil
    config_param :remove_keys,     :string, :default => nil

    def configure(conf)
      super

      if @remove_prefix
        @removed_prefix_string = @remove_prefix + '.'
        @removed_length = @removed_prefix_string.length
      end
      if @add_prefix
        @added_prefix_string = @add_prefix + '.'
      end

      if @keys
        if @remove_keys
          raise ConfigError, "forti_log_parser: 'keys' and 'remove_keys' parameters are exclusive"
        end
        @keys = Hash[@keys.split(',').map {|x| [x, 1] }]
      end
      if @remove_keys
        @remove_keys = Hash[@remove_keys.split(',').map {|x| [x, 1] }]
      end
    end

    def emit(tag, es, chain)
      _tag = tag.clone

      if @remove_prefix and
        ((tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) || tag == @remove_prefix)
        tag = tag[@removed_length..-1] || ''
      end

      if @add_prefix
        tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
      end

      es.each do |time, record|
        time, record = parse(record)
        Engine.emit(tag, time, record)
      end

      chain.next
    end

    def parse(record)
      message = record[@message_key]
      record.delete(@message_key)
      data = message.split(/\s+/, 5).pop
      while data != '' do
        data.gsub!(/^([a-zA-Z0-9_]+)=("[^"]*"|\S+)(:?\s|$)/, '')
        if ($1 == 'date' or $1 == 'time' or
           (@keys and @keys.has_key?($1)) or
           (@remove_keys and not @remove_keys.has_key?($1)) or
           (!@keys and !@remove_keys))
            record[$1] = $2.gsub(/^"(.*)"$/, '\1')
        end
      end

      time = Time.strptime(record["date"] + " " + record["time"], '%Y-%m-%d %H:%M:%S').to_i

      record.delete("date")
      record.delete("time")

      [ time, record ]
    end
  end
end