-
-
Save ThomasBuchinger/1bd636526a198cdd5af58740b12bed60 to your computer and use it in GitHub Desktop.
CloudForms Logstash Config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| file { | |
| path => "/var/www/miq/vmdb/log/automation.log" | |
| codec => multiline { | |
| pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}" | |
| negate => true | |
| what => "previous" | |
| } | |
| } | |
| } | |
| filter { | |
| grok { | |
| match => [ | |
| "message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)" | |
| ] | |
| } | |
| if "_grokparsefailure" not in [tags] { | |
| date { | |
| match => [ "logdate", "ISO8601" ] | |
| } | |
| ruby { | |
| code => | |
| " | |
| logdate = event.get('logdate') | |
| unless logdate.nil? | |
| timestr, usec = logdate.split('.') | |
| # logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec) | |
| time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i | |
| # logger.info('Converted time data: ', 'time_usec' => time_usec) | |
| event.set('time_usec', time_usec.to_s) | |
| end | |
| " | |
| } | |
| grok { | |
| match => { "logmessage" => "AEMethod (?<aemethod>\[[^\]]+\])> Starting" } | |
| add_tag => [ "methodStarting" ] | |
| tag_on_failure => [ ] | |
| } | |
| grok { | |
| match => { "logmessage" => "AEMethod (?<aemethod>\[[^\]]+\])> Ending" } | |
| add_tag => [ "methodEnding" ] | |
| tag_on_failure => [ ] | |
| } | |
| grok { | |
| match => { "logmessage" => "<AEMethod (?<aemethod>[^\[>]+)>.*" } | |
| tag_on_failure => [ ] | |
| } | |
| grok { | |
| match => { "logmessage" => "^Q-task_id\(\[(?<q_task_id>[^\]]+)\]\).*" } | |
| tag_on_failure => [ ] | |
| } | |
| if "methodStarting" in [tags] { | |
| aggregate { | |
| task_id => "%{pid}%{thread}%{aemethod}" | |
| code => " | |
| map['time_usec_start'] = event.get('time_usec') | |
| map['timestamp_start'] = event.get('@timestamp') | |
| " | |
| map_action => "create" | |
| } | |
| } | |
| if "methodEnding" in [tags] { | |
| aggregate { | |
| task_id => "%{pid}%{thread}%{aemethod}" | |
| code => " | |
| event.set('elapsed_time', (event.get('time_usec').to_i - map['time_usec_start'].to_i).to_f / 1000000.0) | |
| event.set('elapsed_timestamp_start', map['timestamp_start']) | |
| " | |
| map_action => "update" | |
| end_of_task => true | |
| timeout => 660 | |
| } | |
| } | |
| ruby { | |
| code => | |
| " | |
| logdate = event.get('logdate') | |
| unless logdate.nil? | |
| timestr, usec = logdate.split('.') | |
| # logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec) | |
| time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i | |
| # logger.info('Converted time data: ', 'time_usec' => time_usec) | |
| event.set('time_usec', time_usec.to_s) | |
| end | |
| aemethod = event.get('aemethod') | |
| thread = event.get('thread') | |
| unless aemethod.nil? or thread.nil? | |
| event.set('elapsed_key', thread.to_s + aemethod.to_s) | |
| end | |
| " | |
| } | |
| } else { | |
| drop { } | |
| } | |
| mutate { | |
| replace => { "message" => "%{logmessage}" } | |
| remove_field => ['logdate' , 'logmessage', 'elapsed_key'] | |
| rename => { "path" => "logfile" } | |
| } | |
| } | |
| output { | |
| # stdout { | |
| # codec => "json_lines" | |
| # } | |
| # file { | |
| # path => "/var/www/miq/vmdb/log/json_evm.log" | |
| # codec => "json_lines" | |
| # } | |
| tcp { | |
| codec => "json_lines" | |
| host => "localhost" | |
| port => 20001 | |
| } | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| file { | |
| path => "/var/www/miq/vmdb/log/evm.log" | |
| codec => multiline { | |
| pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}" | |
| negate => true | |
| what => "previous" | |
| } | |
| } | |
| } | |
| filter { | |
| grok { | |
| match => [ | |
| "message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)" | |
| ] | |
| } | |
| if "_grokparsefailure" not in [tags] { | |
| grok { | |
| match => { "logmessage" => "^Q-task_id\(\[(?<q_task_id>[^\]]+)\]\).*" } | |
| tag_on_failure => [ ] | |
| } | |
| ruby { | |
| code => | |
| " | |
| logdate = event.get('logdate') | |
| unless logdate.nil? | |
| timestr, usec = logdate.split('.') | |
| # logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec) | |
| time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i | |
| # logger.info('Converted time data: ', 'time_usec' => time_usec) | |
| event.set('time_usec', time_usec.to_s) | |
| end | |
| " | |
| } | |
| } else { | |
| drop { } | |
| } | |
| date { | |
| match => [ "logdate", "ISO8601" ] | |
| } | |
| mutate { | |
| replace => { "message" => "%{logmessage}" } | |
| remove_field => ['logdate' , 'logmessage'] | |
| rename => { "path" => "logfile" } | |
| } | |
| } | |
| output { | |
| # stdout { | |
| # codec => "json_lines" | |
| # } | |
| # file { | |
| # path => "/var/www/miq/vmdb/log/json_evm.log" | |
| # codec => "json_lines" | |
| # } | |
| tcp { | |
| codec => "json_lines" | |
| host => "localhost" | |
| port => 20001 | |
| } | |
| } | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| file { | |
| path => "/var/www/miq/vmdb/log/production.log" | |
| codec => multiline { | |
| pattern => "^\[----\] \w, \[%{TIMESTAMP_ISO8601}" | |
| negate => true | |
| what => "previous" | |
| } | |
| } | |
| } | |
| filter { | |
| grok { | |
| match => [ | |
| "message", "\[----\] \w, \[%{TIMESTAMP_ISO8601:logdate}\ \#(?<pid>\d+):(?<thread>[^\]]+)\]\s+(?<ident>\w+) -- : *(?<logmessage>.*)" | |
| ] | |
| } | |
| if "_grokparsefailure" not in [tags] { | |
| grok { | |
| match => { "logmessage" => "^Started (?<http_method>[A-Z]+) \"(?<http_path>[^\"]+)\" for.*" } | |
| add_tag => [ "httpStarting" ] | |
| tag_on_failure => [ ] | |
| } | |
| grok { | |
| match => { "logmessage" => "Completed (?<http_status_code>[0-9]+) [a-zA-Z]+ in (?<elapsed_time>[0-9]+)ms.*" } | |
| add_tag => [ "httpEnding" ] | |
| tag_on_failure => [ ] | |
| } | |
| if "httpStarting" in [tags] { | |
| aggregate { | |
| task_id => "%{pid}%{thread}" | |
| code => " | |
| map['http_method'] = event.get('http_method') | |
| map['http_path'] = event.get('http_path') | |
| " | |
| map_action => "create_or_update" | |
| } | |
| } | |
| if "httpEnding" in [tags] { | |
| aggregate { | |
| task_id => "%{pid}%{thread}" | |
| code => " | |
| event.set('elapsed_time', event.get('elapsed_time').to_f / 1000.0) | |
| event.set('http_method', map['http_method']) | |
| event.set('http_path', map['http_path']) | |
| " | |
| map_action => "update" | |
| end_of_task => true | |
| timeout => 660 | |
| } | |
| } | |
| ruby { | |
| code => | |
| " | |
| logdate = event.get('logdate') | |
| unless logdate.nil? | |
| timestr, usec = logdate.split('.') | |
| # logger.info('Received time data: ', 'timestr' => timestr, 'usec' => usec) | |
| time_usec = Time.parse(timestr).to_i * 1000000 + usec.to_i | |
| # logger.info('Converted time data: ', 'time_usec' => time_usec) | |
| event.set('time_usec', time_usec.to_s) | |
| end | |
| " | |
| } | |
| } else { | |
| drop { } | |
| } | |
| date { | |
| match => [ "logdate", "ISO8601" ] | |
| } | |
| mutate { | |
| replace => { "message" => "%{logmessage}" } | |
| remove_field => ['logdate' , 'logmessage'] | |
| rename => { "path" => "logfile" } | |
| } | |
| } | |
| output { | |
| # stdout { | |
| # codec => "json_lines" | |
| # } | |
| # file { | |
| # path => "/var/www/miq/vmdb/log/json_evm.log" | |
| # codec => "json_lines" | |
| # } | |
| tcp { | |
| codec => "json_lines" | |
| host => "localhost" | |
| port => 20001 | |
| } | |
| } | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "_id": "AWqb6MSu8JS6ixcWpbxb", | |
| "_type": "visualization", | |
| "_source": { | |
| "title": "snapshot_activity", | |
| "visState": "{\"title\":\"snapshot_activity\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='cloudforms-*', q='env:prod AND message:\\\"VM snapshot\\\" AND message:created AND logfile:evm.log').bars(2,false).label(created).title(\\\"snapshot activity\\\"), .es(index='cloudforms-*', q='env:prod AND message:\\\"VM snapshot\\\" AND message:removed AND logfile:evm.log').bars(2,false).multiply(-1).label(removed), .es(index='cloudforms-*',q='env:prod AND message:Handsoap AND logfile:evm.log').bars(2,false).label('remove errors')\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}", | |
| "uiStateJSON": "{}", | |
| "description": "", | |
| "version": 1, | |
| "kibanaSavedObjectMeta": { | |
| "searchSourceJSON": "{\"query\":{\"match_all\":{}},\"filter\":[]}" | |
| } | |
| } | |
| }, | |
| { | |
| "_id": "AWALyZS1Ux89uWFGMIAd", | |
| "_type": "visualization", | |
| "_source": { | |
| "title": "Prod Provisioning Requests", | |
| "visState": "{\"title\":\"Prod Provisioning Requests\",\"type\":\"histogram\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per hour\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", | |
| "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", | |
| "description": "", | |
| "savedSearchId": "AV-z9t5OUx89uWFGF1ql", | |
| "version": 1, | |
| "kibanaSavedObjectMeta": { | |
| "searchSourceJSON": "{\"filter\":[]}" | |
| } | |
| } | |
| }, | |
| { | |
| "_id": "AWQCJVoo8JS6ixcWsWS_", | |
| "_type": "visualization", | |
| "_source": { | |
| "title": "Automate dialog engine timings", | |
| "visState": "{\"title\":\"Automate dialog engine timings\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":500},\"position\":\"left\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"aemethod: Descending\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"4\",\"label\":\"Median elapsed_time\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"},{\"data\":{\"id\":\"5\",\"label\":\"Standard Deviation of elapsed_time\"},\"drawLinesBetweenPoints\":false,\"lineWidth\":10,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"BottomAxis-1\",\"position\":\"bottom\",\"scale\":{\"max\":20,\"min\":-1,\"mode\":\"normal\",\"setYExtents\":true,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Median elapsed_time\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"aemethod\",\"size\":30,\"orderAgg\":{\"id\":\"3-orderAgg\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"elapsed_time\"}},\"order\":\"desc\",\"orderBy\":\"custom\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\",\"percents\":[50]}},{\"id\":\"5\",\"enabled\":true,\"type\":\"std_dev\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\"}}],\"listeners\":{}}", | |
| "uiStateJSON": "{\"vis\":{\"colors\":{\"50th percentile of elapsed_time\":\"#6ED0E0\",\"50th percentile of median elapsed_time\":\"#64B0C8\",\"median elapsed_time\":\"#447EBC\"},\"legendOpen\":false}}", | |
| "description": "", | |
| "version": 1, | |
| "kibanaSavedObjectMeta": { | |
| "searchSourceJSON": "{\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"query\":{\"query_string\":{\"query\":\"logfile:automation.log AND env:prod AND _exists_:aemethod AND _exists_:elapsed_time AND (aemethod:*ServiceDialogs*)\",\"analyze_wildcard\":true}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"key\":\"elapsed_time\",\"negate\":false,\"type\":\"range\",\"value\":\"-∞ to 600\"},\"range\":{\"elapsed_time\":{\"gte\":null,\"lt\":600}}}]}" | |
| } | |
| } | |
| }, | |
| { | |
| "_id": "AWPeZnGB8JS6ixcWB49Q", | |
| "_type": "visualization", | |
| "_source": { | |
| "title": "Long running http requests - timings", | |
| "visState": "{\"title\":\"Long running http requests - timings\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":500,\"filter\":false},\"position\":\"left\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"http_path: Descending\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"4\",\"label\":\"Median elapsed_time\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"step-after\",\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"},{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":false,\"showCircles\":true,\"interpolate\":\"step-after\",\"lineWidth\":10,\"data\":{\"id\":\"5\",\"label\":\"Standard Deviation of elapsed_time\"},\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"BottomAxis-1\",\"position\":\"bottom\",\"scale\":{\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Median elapsed_time\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"http_path\",\"size\":30,\"orderAgg\":{\"id\":\"3-orderAgg\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"elapsed_time\"}},\"order\":\"desc\",\"orderBy\":\"custom\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\",\"percents\":[50]}},{\"id\":\"5\",\"enabled\":true,\"type\":\"std_dev\",\"schema\":\"metric\",\"params\":{\"field\":\"elapsed_time\"}}],\"listeners\":{}}", | |
| "uiStateJSON": "{\"vis\":{\"colors\":{\"50th percentile of median elapsed_time\":\"#64B0C8\",\"median elapsed_time\":\"#447EBC\",\"50th percentile of elapsed_time\":\"#65C5DB\"},\"legendOpen\":true}}", | |
| "description": "", | |
| "version": 1, | |
| "kibanaSavedObjectMeta": { | |
| "searchSourceJSON": "{\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"query\":{\"query_string\":{\"query\":\"logfile:production.log AND env:prod AND _exists_:http_path AND http_method:POST\",\"analyze_wildcard\":true}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"AWPUHqE68JS6ixcWzaxk\",\"key\":\"elapsed_time\",\"negate\":false,\"type\":\"range\",\"value\":\"2 to +∞\"},\"range\":{\"elapsed_time\":{\"gte\":2,\"lt\":null}}}]}" | |
| } | |
| } | |
| } | |
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This file is where you define your pipelines. You can define multiple. | |
| # For more information on multiple pipelines, see the documentation: | |
| # https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html | |
| - pipeline.id: main | |
| path.config: "/etc/logstash/conf.d/*.conf" | |
| - pipeline.id: cloudforms-evm | |
| path.config: "/etc/logstash/cloudforms/cloudforms-evm.conf" | |
| pipeline.workers: 4 | |
| - pipeline.id: cloudforms-automation | |
| path.config: "/etc/logstash/cloudforms/cloudforms-automation.conf" | |
| pipeline.workers: 1 | |
| - pipeline.id: cloudforms-production | |
| path.config: "/etc/logstash/cloudforms/cloudforms-production.conf" | |
| pipeline.workers: 1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment