Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
CloudForms Logstash Config
input {
file {
path => "/var/www/miq/vmdb/log/automation.log"
codec => multiline {
pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
}
}
}
filter {
grok {
match => [
"message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)"
]
}
if "_grokparsefailure" not in [tags] {
date {
match => [ "logdate", "ISO8601" ]
}
ruby {
code =>
"
logdate = event.get('logdate')
unless logdate.nil?
timestr, usec = logdate.split('.')
# logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec)
time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i
# logger.info('Converted time data: ', 'time_usec' => time_usec)
event.set('time_usec', time_usec.to_s)
end
"
}
grok {
match => { "logmessage" => "AEMethod (?<aemethod>\[[^\]]+\])> Starting" }
add_tag => [ "methodStarting" ]
tag_on_failure => [ ]
}
grok {
match => { "logmessage" => "AEMethod (?<aemethod>\[[^\]]+\])> Ending" }
add_tag => [ "methodEnding" ]
tag_on_failure => [ ]
}
grok {
match => { "logmessage" => "<AEMethod (?<aemethod>[^\[>]+)>.*" }
tag_on_failure => [ ]
}
grok {
match => { "logmessage" => "^Q-task_id\(\[(?<q_task_id>[^\]]+)\]\).*" }
tag_on_failure => [ ]
}
if "methodStarting" in [tags] {
aggregate {
task_id => "%{pid}%{thread}%{aemethod}"
code => "
map['time_usec_start'] = event.get('time_usec')
map['timestamp_start'] = event.get('@timestamp')
"
map_action => "create"
}
}
if "methodEnding" in [tags] {
aggregate {
task_id => "%{pid}%{thread}%{aemethod}"
code => "
event.set('elapsed_time', (event.get('time_usec').to_i - map['time_usec_start'].to_i).to_f / 1000000.0)
event.set('elapsed_timestamp_start', map['timestamp_start'])
"
map_action => "update"
end_of_task => true
timeout => 660
}
}
ruby {
code =>
"
logdate = event.get('logdate')
unless logdate.nil?
timestr, usec = logdate.split('.')
# logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec)
time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i
# logger.info('Converted time data: ', 'time_usec' => time_usec)
event.set('time_usec', time_usec.to_s)
end
aemethod = event.get('aemethod')
thread = event.get('thread')
unless aemethod.nil? or thread.nil?
event.set('elapsed_key', thread.to_s + aemethod.to_s)
end
"
}
} else {
drop { }
}
mutate {
replace => { "message" => "%{logmessage}" }
remove_field => ['logdate' , 'logmessage', 'elapsed_key']
rename => { "path" => "logfile" }
}
}
output {
# stdout {
# codec => "json_lines"
# }
# file {
# path => "/var/www/miq/vmdb/log/json_evm.log"
# codec => "json_lines"
# }
tcp {
codec => "json_lines"
host => "localhost"
port => 20001
}
}
input {
file {
path => "/var/www/miq/vmdb/log/evm.log"
codec => multiline {
pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
}
}
}
filter {
grok {
match => [
"message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)"
]
}
if "_grokparsefailure" not in [tags] {
grok {
match => { "logmessage" => "^Q-task_id\(\[(?<q_task_id>[^\]]+)\]\).*" }
tag_on_failure => [ ]
}
ruby {
code =>
"
logdate = event.get('logdate')
unless logdate.nil?
timestr, usec = logdate.split('.')
# logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec)
time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i
# logger.info('Converted time data: ', 'time_usec' => time_usec)
event.set('time_usec', time_usec.to_s)
end
"
}
} else {
drop { }
}
date {
match => [ "logdate", "ISO8601" ]
}
mutate {
replace => { "message" => "%{logmessage}" }
remove_field => ['logdate' , 'logmessage']
rename => { "path" => "logfile" }
}
}
output {
# stdout {
# codec => "json_lines"
# }
# file {
# path => "/var/www/miq/vmdb/log/json_evm.log"
# codec => "json_lines"
# }
tcp {
codec => "json_lines"
host => "localhost"
port => 20001
}
}
input {
file {
path => "/var/www/miq/vmdb/log/production.log"
codec => multiline {
pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}"
negate => true
what => "previous"
}
}
}
filter {
grok {
match => [
"message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)"
]
}
if "_grokparsefailure" not in [tags] {
grok {
match => { "logmessage" => "^Started (?<http_method>[A-Z]+) \"(?<http_path>[^\"]+)\" for.*" }
add_tag => [ "httpStarting" ]
tag_on_failure => [ ]
}
grok {
match => { "logmessage" => "Completed (?<http_status_code>[0-9]+) [a-zA-Z]+ in (?<elapsed_time>[0-9]+)ms.*" }
add_tag => [ "httpEnding" ]
tag_on_failure => [ ]
}
if "httpStarting" in [tags] {
aggregate {
task_id => "%{pid}%{thread}"
code => "
map['http_method'] = event.get('http_method')
map['http_path'] = event.get('http_path')
"
map_action => "create_or_update"
}
}
if "httpEnding" in [tags] {
aggregate {
task_id => "%{pid}%{thread}"
code => "
event.set('elapsed_time', event.get('elapsed_time').to_f / 1000.0)
event.set('http_method', map['http_method'])
event.set('http_path', map['http_path'])
"
map_action => "update"
end_of_task => true
timeout => 660
}
}
ruby {
code =>
"
logdate = event.get('logdate')
unless logdate.nil?
timestr, usec = logdate.split('.')
# logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec)
time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i
# logger.info('Converted time data: ', 'time_usec' => time_usec)
event.set('time_usec', time_usec.to_s)
end
"
}
} else {
drop { }
}
date {
match => [ "logdate", "ISO8601" ]
}
mutate {
replace => { "message" => "%{logmessage}" }
remove_field => ['logdate' , 'logmessage']
rename => { "path" => "logfile" }
}
}
output {
# stdout {
# codec => "json_lines"
# }
# file {
# path => "/var/www/miq/vmdb/log/json_evm.log"
# codec => "json_lines"
# }
tcp {
codec => "json_lines"
host => "localhost"
port => 20001
}
}
[
{
"_id": "AWqb6MSu8JS6ixcWpbxb",
"_type": "visualization",
"_source": {
"title": "snapshot_activity",
"visState": "{\"title\":\"snapshot_activity\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='cloudforms-*', q='env:prod AND message:\\\"VM snapshot\\\" AND message:created AND logfile:evm.log').bars(2,false).label(created).title(\\\"snapshot activity\\\"), .es(index='cloudforms-*', q='env:prod AND message:\\\"VM snapshot\\\" AND message:removed AND logfile:evm.log').bars(2,false).multiply(-1).label(removed), .es(index='cloudforms-*',q='env:prod AND message:Handsoap AND logfile:evm.log').bars(2,false).label('remove errors')\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"match_all\":{}},\"filter\":[]}"
}
}
},
{
"_id": "AWALyZS1Ux89uWFGMIAd",
"_type": "visualization",
"_source": {
"title": "Prod Provisioning Requests",
"visState": "{\"title\":\"Prod Provisioning Requests\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per hour\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"legendOpen\":true}}",
"description": "",
"savedSearchId": "AV-z9t5OUx89uWFGF1ql",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}
},
{
"_id": "AWQCJVoo8JS6ixcWsWS_",
"_type": "visualization",
"_source": {
"title": "Automate dialog engine timings",
"visState": "{\"title\":\"Automate dialog engine timings\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":500},\"position\":\"left\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"aemethod: Descending\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"4\",\"label\":\"Median elapsed_time\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"},{\"data\":{\"id\":\"5\",\"label\":\"Standard Deviation of elapsed_time\"},\"drawLinesBetweenPoints\":false,\"lineWidth\":10,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"BottomAxis-1\",\"position\":\"bottom\",\"scale\":{\"max\":20,\"min\":-1,\"mode\":\"normal\",\"setYExtents\":true,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Median elapsed_time\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"aemethod\",\"size\":30,\"orderAgg\":{\"id\":\"3-orderAgg\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"elapsed_time\"}},\"order\":\"desc\",\"orderBy\":\"custom\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\",\"percents\":[50]}},{\"id\":\"5\",\"enabled\":true,\"type\":\"std_dev\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"50th percentile of elapsed_time\":\"#6ED0E0\",\"50th percentile of median elapsed_time\":\"#64B0C8\",\"median elapsed_time\":\"#447EBC\"},\"legendOpen\":false}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"query\":{\"query_string\":{\"query\":\"logfile:automation.log AND env:prod AND _exists_:aemethod AND _exists_:elapsed_time AND (aemethod:*ServiceDialogs*)\",\"analyze_wildcard\":true}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"key\":\"elapsed_time\",\"negate\":false,\"type\":\"range\",\"value\":\"-∞ to 600\"},\"range\":{\"elapsed_time\":{\"gte\":null,\"lt\":600}}}]}"
}
}
},
{
"_id": "AWPeZnGB8JS6ixcWB49Q",
"_type": "visualization",
"_source": {
"title": "Long running http requests - timings",
"visState": "{\"title\":\"Long running http requests - timings\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":500,\"filter\":false},\"position\":\"left\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"http_path: Descending\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"4\",\"label\":\"Median elapsed_time\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"step-after\",\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"},{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":false,\"showCircles\":true,\"interpolate\":\"step-after\",\"lineWidth\":10,\"data\":{\"id\":\"5\",\"label\":\"Standard Deviation of elapsed_time\"},\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"BottomAxis-1\",\"position\":\"bottom\",\"scale\":{\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Median elapsed_time\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"http_path\",\"size\":30,\"orderAgg\":{\"id\":\"3-orderAgg\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"elapsed_time\"}},\"order\":\"desc\",\"orderBy\":\"custom\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\",\"percents\":[50]}},{\"id\":\"5\",\"enabled\":true,\"type\":\"std_dev\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"50th percentile of median elapsed_time\":\"#64B0C8\",\"median elapsed_time\":\"#447EBC\",\"50th percentile of elapsed_time\":\"#65C5DB\"},\"legendOpen\":true}}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"query\":{\"query_string\":{\"query\":\"logfile:production.log AND env:prod AND _exists_:http_path AND http_method:POST\",\"analyze_wildcard\":true}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"key\":\"elapsed_time\",\"negate\":false,\"type\":\"range\",\"value\":\"2 to +∞\"},\"range\":{\"elapsed_time\":{\"gte\":2,\"lt\":null}}}]}"
}
}
}
]
# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
- pipeline.id: main
path.config: "/etc/logstash/conf.d/*.conf"
- pipeline.id: cloudforms-evm
path.config: "/etc/logstash/cloudforms/cloudforms-evm.conf"
pipeline.workers: 4
- pipeline.id: cloudforms-automation
path.config: "/etc/logstash/cloudforms/cloudforms-automation.conf"
pipeline.workers: 1
- pipeline.id: cloudforms-production
path.config: "/etc/logstash/cloudforms/cloudforms-production.conf"
pipeline.workers: 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.