Skip to content

Instantly share code, notes, and snippets.

@yteraoka

yteraoka/bench.rb

Last active Dec 25, 2015
Embed
What would you like to do?
in_tail_asis とか in_tail の none parser で読み込んだ FortiGate の syslog を parse する。 FortiGate のログは項目が多いので keys か remove_keys で必要なものだけに絞る。 loop で gsub! するのは非常に重いので書き直した。
require 'benchmark'
lines = []
f = open(ARGV[0])
while line = f.gets
lines.push line.chomp
break
end
f.close
n = 10000
Benchmark.bm do |x|
x.report("A: ") {
(1..n).each {
lines.each { |line|
record = {}
data = line.split(/\s+/, 5).pop
while data != '' do
data.gsub!(/^([a-zA-Z0-9_]+)=("[^"]*"|\S+)(:?\s|$)/, '')
record[$1] = $2.gsub(/^"(.*)"$/, '\1')
end
#puts record
}
}
}
x.report("B: ") {
(1..n).each {
lines.each { |line|
record = {}
data = line.split(/\s+/, 5).pop
data.gsub(/\G[a-zA-Z0-9_]+=(:?"[^"]*"|\S+)(:?\s|$)/) { |kv|
(k, v) = kv.strip.split(/=/, 2)
record[k] = v.gsub(/^"(.*)"$/, '\1')
}
#puts record
}
}
}
end
# ./bench.rb forti.1000
user system total real
A: 15.190000 0.000000 15.190000 ( 15.193379)
B: 3.870000 0.000000 3.870000 ( 3.871790)
module Fluent
class FortigateSyslogParseOutput < Output
Fluent::Plugin.register_output('forti_log_parser', self)
config_param :remove_prefix, :string, :default => nil
config_param :add_prefix, :string, :default => nil
config_param :message_key, :string, :default => 'message'
config_param :keys, :string, :default => nil
config_param :remove_keys, :string, :default => nil
def configure(conf)
super
if @remove_prefix
@removed_prefix_string = @remove_prefix + '.'
@removed_length = @removed_prefix_string.length
end
if @add_prefix
@added_prefix_string = @add_prefix + '.'
end
if @keys
if @remove_keys
raise ConfigError, "forti_log_parser: 'keys' and 'remove_keys' parameters are exclusive"
end
@keys = Hash[@keys.split(',').map {|x| [x, 1] }]
end
if @remove_keys
@remove_keys = Hash[@remove_keys.split(',').map {|x| [x, 1] }]
end
end
def emit(tag, es, chain)
_tag = tag.clone
if @remove_prefix and
((tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) || tag == @remove_prefix)
tag = tag[@removed_length..-1] || ''
end
if @add_prefix
tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
end
es.each do |time, record|
time, record = parse(record)
Engine.emit(tag, time, record)
end
chain.next
end
def parse(record)
message = record[@message_key]
record.delete(@message_key)
data = message.split(/\s+/, 5).pop
data.gsub(/\G[a-zA-Z0-9_]+=(:?"[^"]*"|\S+)(:?\s|$)/) { |kv|
(k, v) = kv.strip.split(/=/, 2)
if (k == 'date' or k == 'time' or
(@keys and @keys.has_key?(k)) or
(@remove_keys and not @remove_keys.has_key?(k)) or
(!@keys and !@remove_keys))
record[k] = v.gsub(/^"(.*)"$/, '\1')
end
}
time = Time.strptime(record["date"] + " " + record["time"], '%Y-%m-%d %H:%M:%S').to_i
record.delete("date")
record.delete("time")
[ time, record ]
end
end
end
module Fluent
class FortigateSyslogParseOutput < Output
Fluent::Plugin.register_output('forti_log_parser', self)
config_param :remove_prefix, :string, :default => nil
config_param :add_prefix, :string, :default => nil
config_param :message_key, :string, :default => 'message'
config_param :keys, :string, :default => nil
config_param :remove_keys, :string, :default => nil
def configure(conf)
super
if @remove_prefix
@removed_prefix_string = @remove_prefix + '.'
@removed_length = @removed_prefix_string.length
end
if @add_prefix
@added_prefix_string = @add_prefix + '.'
end
if @keys
if @remove_keys
raise ConfigError, "forti_log_parser: 'keys' and 'remove_keys' parameters are exclusive"
end
@keys = Hash[@keys.split(',').map {|x| [x, 1] }]
end
if @remove_keys
@remove_keys = Hash[@remove_keys.split(',').map {|x| [x, 1] }]
end
end
def emit(tag, es, chain)
_tag = tag.clone
if @remove_prefix and
((tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) || tag == @remove_prefix)
tag = tag[@removed_length..-1] || ''
end
if @add_prefix
tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
end
es.each do |time, record|
time, record = parse(record)
Engine.emit(tag, time, record)
end
chain.next
end
def parse(record)
message = record[@message_key]
record.delete(@message_key)
data = message.split(/\s+/, 5).pop
while data != '' do
data.gsub!(/^([a-zA-Z0-9_]+)=("[^"]*"|\S+)(:?\s|$)/, '')
if ($1 == 'date' or $1 == 'time' or
(@keys and @keys.has_key?($1)) or
(@remove_keys and not @remove_keys.has_key?($1)) or
(!@keys and !@remove_keys))
record[$1] = $2.gsub(/^"(.*)"$/, '\1')
end
end
time = Time.strptime(record["date"] + " " + record["time"], '%Y-%m-%d %H:%M:%S').to_i
record.delete("date")
record.delete("time")
[ time, record ]
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.