Skip to content

Instantly share code, notes, and snippets.

@jehuty0shift

jehuty0shift/filter.logstash

Last active January 26, 2023 01:25
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save jehuty0shift/c98f8e400962981887a28771037b3c34 to your computer and use it in GitHub Desktop.
Save jehuty0shift/c98f8e400962981887a28771037b3c34 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
filter.logstash
if [event][module] == "apache" {
if [fileset][name] == "access" {
grok {
match => { "message" => [
"%{IPORHOST:[source][address]} - %{DATA:[user][name]} \[%{HTTPDATE:[apache][access][time]}\] \"(?:%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]:float}|-)?\" %{NUMBER:[http][response][status_code]:int} (?:%{NUMBER:[http][response][body][bytes]:int}|-)( \"%{DATA:[http][request][referrer]}\")?( \"%{DATA:[user_agent][original]}\")?",
"%{IPORHOST:[source][address]} - %{DATA:[user][name]} \[%{HTTPDATE:[apache][access][time]}\] \"-\" %{NUMBER:[http][response][status_code]:int} -",
"\[%{HTTPDATE:[apache][access][time]}\] %{IPORHOST:[source][address]} %{DATA:[apache][access][ssl][protocol]} %{DATA:[apache][access][ssl][cipher]} \"%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]:float}\" %{NUMBER:[http][response][body][bytes]:int}"]
}
remove_field => [ "message" ]
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [source][address] {
grok {
match => { "[source][address]" => [ "^(%{IP:[source][ip]}|%{HOSTNAME:[source][domain]})$" ] }
}
}
if [event][timezone] {
date {
match => [ "[apache][access][time]", "dd/MMM/yyyy:H:m:s Z" ]
timezone => "%{[event][timezone]}"
remove_field => "[apache][access][time]"
}
} else {
date {
match => [ "[apache][access][time]", "dd/MMM/yyyy:H:m:s Z" ]
remove_field => "[apache][access][time]"
}
}
if [user_agent][original] {
useragent {
source => "[user_agent][original]"
}
}
}
} else if [fileset][name] == "error" {
grok {
match => { "message" => [
"\[%{APACHE_TIME:[apache][error][timestamp]}\] \[%{LOGLEVEL:[log][level]}\]( \[client %{IPORHOST:[source][address]}(:%{POSINT:[source][port]:int})?\])? %{GREEDYDATA:message}",
"\[%{APACHE_TIME:[apache][error][timestamp]}\] \[%{DATA:[apache][error][module]}:%{LOGLEVEL:[log][level]}\] \[pid %{NUMBER:[process][pid]:int}(:tid %{NUMBER:[process][thread][id]:int})?\]( \[client %{IPORHOST:[source][address]}(:%{POSINT:[source][port]:int})?\])? %{GREEDYDATA:message}" ]
}
pattern_definitions => {
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
}
}
if ("_grokparsefailure" not in [tags]) {
if [event][timezone] {
date {
match => [ "[apache][error][timestamp]", "EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy" ]
timezone => "%{event][timezone]}"
remove_field => "[apache][error][timestamp]"
}
} else {
date {
match => [ "[apache][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
remove_field => "[apache][error][timestamp]"
}
}
if [source][address] {
grok {
match => { "[source][address]" => [ "^(%{IP:[source][ip]}|%{HOSTNAME:[source][domain]})$" ] }
}
}
}
}
} else if [event][module] == "auditd" {
if [fileset][name] == "log" {
grok {
match => { "message" => [
"%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:[auditd][log][kv]} old auid=%{NUMBER:[auditd][log][old_auid]:int} new auid=%{NUMBER:[auditd][log][new_auid]:int} old ses=%{NUMBER:[auditd][log][old_ses]:int} new ses=%{NUMBER:[auditd][log][new_ses]:int}",
"%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:[auditd][log][kv]} msg=['\"](%{DATA:[auditd][log][msg]}\s+)?%{AUDIT_KEY_VALUES:[auditd][log][sub_kv]}['\"]",
"%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:[auditd][log][kv]}",
"%{AUDIT_PREFIX}",
"%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:[auditd][log][kv]}"]
}
pattern_definitions => {
"AUDIT_TYPE" => "^type=%{NOTSPACE:[auditd][log][record_type]}"
"AUDIT_PREFIX" => "%{AUDIT_TYPE} msg=audit\(%{NUMBER:[auditd][log][epoch]:int}:%{NUMBER:[auditd][log][sequence]:int}\):(%{DATA})?"
"AUDIT_KEY_VALUES" => "%{WORD}=%{GREEDYDATA}"
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
remove_field => [ "message" ]
}
if [auditd][log][kv] {
kv {
source => "[auditd][log][kv]"
target => "[auditd][log]"
field_split => "\s+"
value_split => "="
remove_field => "[auditd][log][kv]"
}
}
if [auditd][log][sub_kv] {
kv {
source => "[auditd][log][sub_kv]"
target => "[auditd][log]"
field_split => "\s+"
value_split => "="
remove_field => "[auditd][log][sub_kv]"
}
}
if [auditd][log][epoch] {
date {
match => [ "[auditd][log][epoch]", "UNIX" ]
remove_field => "[auditd][log][epoch]"
}
}
mutate {
rename => {
"[auditd][log][arch]" => "[host][architecture]"
"[auditd][log][acct]" => "[user][name]"
"[auditd][log][uid]" => "[user][id]"
"[auditd][log][gid]" => "[user][group][id]"
"[auditd][log][agid]" => "[user][audit][group][id]"
"[auditd][log][auid]" => "[user][audit][id]"
"[auditd][log][fsgid]" => "[user][filesystem][group][id]"
"[auditd][log][fsuid]" => "[user][filesystem][id]"
"[auditd][log][egid]" => "[user][effective][group][id]"
"[auditd][log][euid]" => "[user][effective][id]"
"[auditd][log][sgid]" => "[user][saved][group][id]"
"[auditd][log][suid]" => "[user][saved][id]"
"[auditd][log][ogid]" => "[user][owner][group][id]"
"[auditd][log][ouid]" => "[user][owner][id]"
"[auditd][log][comm]" => "[process][name]"
"[auditd][log][exe]" => "[process][executable]"
"[auditd][log][pid]" => "[process][pid]"
"[auditd][log][ppid]" => "[process][ppid]"
"[auditd][log][cmd]" => "[process][args]"
"[auditd][log][terminal]" => "[user][terminal]"
"[auditd][log][msg]" => "[message]"
"[auditd][log][res]" => "[event][outcome]"
"[auditd][log][record_type]" => "[event][action]"
"[auditd][log][src]" => "[source][address]"
"[auditd][log][dst]" => "[destination][address]"
}
convert => {
"[process][pid]" => "integer"
"[process][ppid]" => "integer"
}
split => { "[process][args]" => "\s+" }
lowercase => [ "[event][action]" ]
}
if [source][address] {
grok {
match => { "[source][address]" => [ "^%{IP:[source][ip]}$" ] }
}
}
}
}
} else if [event][module] == "elasticsearch" {
if [fileset][name] == "audit" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[elasticsearch][audit]"
}
if [elasticsearch][audit][type] {
if [elasticsearch][audit][type] != "audit" {
drop { }
}
}
de_dot {
fields => [ "[elasticsearch][audit][event.action]", "[elasticsearch][audit][event.type]", "[elasticsearch][audit][origin.address]", "[elasticsearch][audit][url.path]", "[elasticsearch][audit][url.query]", "[elasticsearch][audit][node.id]", "[elasticsearch][audit][node.name]", "[elasticsearch][audit][user.name]", "[elasticsearch][audit][request.method]", "[elasticsearch][audit][request.body]", "[elasticsearch][audit][cluster.name]" ]
nested => true
}
mutate {
remove_field => [ "[elasticsearch][audit][type]" ]
rename => {
"[elasticsearch][audit][level]" => "[log][level]"
"[elasticsearch][audit][timestamp]" => "[elasticsearch][audit][@timestamp]"
"[elasticsearch][audit][event][type]" => "[elasticsearch][audit][layer]"
"[elasticsearch][audit][cluster][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][audit][user][name]" => "[user][name]"
"[elasticsearch][audit][request][method]" => "[http][request][method]"
"[elasticsearch][audit][request][body]" => "[http][request][body][content]"
}
copy => { "[elasticsearch][audit][node]" => "[elasticsearch][node]" }
remove_field => [ "[elasticsearch][audit][node]" ]
}
if [elasticsearch][audit][url][path] {
if [elasticsearch][audit][url][query] {
mutate {
add_field => { "[url][original]" => "%{[elasticsearch][audit][url][path]}?%{[elasticsearch][audit][url][query]}" }
remove_field => [ "[elasticsearch][audit][url][path]", "[elasticsearch][audit][url][query]" ]
}
} else {
mutate {
add_field => { "[url][original]" => "%{[elasticsearch][audit][url][path]}" }
remove_field => [ "[elasticsearch][audit][url][path]" ]
}
}
}
grok {
match => { "[elasticsearch][audit][origin][address]" => [
"\[%{IPORHOST:[source][ip]}\]:%{INT:[source][port]:int}",
"%{IPORHOST:[source][ip]}:%{INT:[source][port]:int}" ]
}
}
mutate {
rename => { "[elasticsearch][audit][origin][address]" => "[source][address]" }
}
} else {
grok {
match => { "message" => [ "%{ES_TIMESTAMP}\s*%{ES_NODE_NAME}\s*%{ES_AUDIT_LAYER}\s*%{ES_AUDIT_EVENT_TYPE}\s*%{ES_AUDIT_ORIGIN_TYPE},?\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\s*%{ES_AUDIT_PRINCIPAL},?\s*%{ES_AUDIT_REALM},?\s*%{ES_AUDIT_ROLES},?\s*%{ES_AUDIT_ACTION},?\s*%{ES_AUDIT_INDICES},?\s*%{ES_AUDIT_URI},?\s*%{ES_AUDIT_URI_PARAMS},?\s*%{ES_AUDIT_REQUEST},?\s*%{ES_AUDIT_REQUEST_BODY},?" ]
}
pattern_definitions => {
"ES_TIMESTAMP" => "\[%{TIMESTAMP_ISO8601:[elasticsearch][audit][@timestamp]}\]"
"ES_NODE_NAME" => "(\[%{DATA:[elasticsearch][node][name]}\])?"
"ES_AUDIT_LAYER" => "\[%{WORD:[elasticsearch][audit][layer]}\]"
"ES_AUDIT_EVENT_TYPE" => "\[%{WORD:[elasticsearch][audit][event_type]}\]"
"ES_AUDIT_ORIGIN_TYPE" => "(origin_type\=\[%{WORD:[elasticsearch][audit][origin][type]}\])?"
"ES_AUDIT_ORIGIN_ADDRESS" => "(origin_address\=\[%{IPORHOST:[source][ip]}\])?"
"ES_AUDIT_PRINCIPAL" => "(principal\=\[%{DATA:[user][name]}\])?"
"ES_AUDIT_REALM" => "(realm\=\[%{WORD:[elasticsearch][audit][realm]}\])?"
"ES_AUDIT_ROLES" => "(roles\=\[%{DATA:[elasticsearch][audit][user][roles]}\])?"
"ES_AUDIT_ACTION" => "(action\=\[%{DATA:[elasticsearch][audit][action]}(\[%{DATA:[elasticsearch][audit][sub_action]}\])?\])?"
"ES_AUDIT_URI" => "(uri=\[%{DATA:[url][original]}\])?"
"ES_AUDIT_URI_PARAMS" => "(params=\[%{DATA:[elasticsearch][audit][url][params]}\])?"
"ES_AUDIT_INDICES" => "(indices\=\[%{DATA:[elasticsearch][audit][indices]}\])?"
"ES_AUDIT_REQUEST" => "(request\=\[%{WORD:[elasticsearch][audit][request][name]}\])?"
"ES_AUDIT_REQUEST_BODY" => "(request_body\=\[%{DATA:[http][request][body][content]}\])?"
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
split => { "[elasticsearch][audit][indices]" => "," }
split => { "[elasticsearch][audit][user][roles]" => "," }
}
if [elasticsearch][audit][sub_action] {
mutate {
merge => { "[elasticsearch][audit][action]" => "[elasticsearch][audit][sub_action]" }
remove_field => "[elasticsearch][audit][sub_action]"
}
}
}
}
if [elasticsearch][audit][@timestamp] {
if [event][timezone] {
date {
match => [ "[elasticsearch][audit][@timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[elasticsearch][audit][@timestamp]"
}
} else {
date {
match => [ "[elasticsearch][audit][@timestamp]", "ISO8601" ]
remove_field => "[elasticsearch][audit][@timestamp]"
}
}
}
mutate {
remove_field => [ "first_char" ]
}
}
} else if [fileset][name] == "deprecation" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[elasticsearch][deprecation]"
}
if [elasticsearch][deprecation][type] != "deprecation" {
drop { }
}
de_dot {
fields => [ "[elasticsearch][deprecation][cluster.name]", "[elasticsearch][deprecation][node.name]", "[elasticsearch][deprecation][cluster.uuid]", "[elasticsearch][deprecation][node.id]" ]
nested => true
}
mutate {
remove_field => [ "[elasticsearch][deprecation][type]" ]
rename => {
"[elasticsearch][deprecation][level]" => "[log][level]"
"[elasticsearch][deprecation][component]" => "[elasticsearch][component]"
"[elasticsearch][deprecation][cluster][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][deprecation][node][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][deprecation][cluster][uuid]" => "[elasticsearch][cluster][uuid]"
"[elasticsearch][deprecation][node][id]" => "[elasticsearch][node][id]"
"[elasticsearch][deprecation][message]" => "message"
}
}
} else {
grok {
match => { "message" => [ "\[%{TIMESTAMP_ISO8601:[elasticsearch][deprecation][timestamp]}\]\[%{LOGLEVEL:[log][level]}%{SPACE}*\]\[%{DATA:[elasticsearch][component]}%{SPACE}*\] %{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
}
}
}
if [event][timezone] {
date {
match => [ "[elasticsearch][deprecation][timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[elasticsearch][deprecation][timestamp]"
}
} else {
date {
match => [ "[elasticsearch][deprecation][timestamp]", "ISO8601" ]
remove_field => "[elasticsearch][deprecation][timestamp]"
}
}
mutate {
remove_field => [ "first_char" ]
}
}
} else if [fileset][name] == "gc" {
grok {
match => { "message" => [
"(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were stopped: %{BASE10NUM:[elasticsearch][gc][threads_total_stop_time_sec]:float} seconds, Stopping threads took: %{BASE10NUM:[elasticsearch][gc][stopping_threads_time_sec]:float} seconds",
"(?:%{JVM8HEADER}) \[GC \(%{DATA:[elasticsearch][gc][phase][name]}\) \[YG occupancy: %{BASE10NUM:[elasticsearch][gc][young_gen][used_kb]:int} K \(%{BASE10NUM:[elasticsearch][gc][young_gen][size_kb]:int} K\)\]%{BASE10NUM}: \[Rescan \(parallel\) , %{BASE10NUM:[elasticsearch][gc][phase][parallel_rescan_time_sec]:float} secs\]%{BASE10NUM}: \[weak refs processing, %{BASE10NUM:[elasticsearch][gc][phase][weak_refs_processing_time_sec]:float} secs\]%{BASE10NUM}: \[class unloading, %{BASE10NUM:[elasticsearch][gc][phase][class_unload_time_sec]:float} secs\]%{BASE10NUM}: \[scrub symbol table, %{BASE10NUM:[elasticsearch][gc][phase][scrub_symbol_table_time_sec]:float} secs\]%{BASE10NUM}: \[scrub string table, %{BASE10NUM:[elasticsearch][gc][phase][scrub_string_table_time_sec]:float} secs\]\[1 CMS-remark: %{BASE10NUM:[elasticsearch][gc][old_gen][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][old_gen][size_kb]:int}K\)\] %{BASE10NUM:[elasticsearch][gc][heap][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][heap][size_kb]:int}K\), %{BASE10NUM:[elasticsearch][gc][phase][duration_sec]:float} secs\] %{PROCTIME}",
"(?:%{JVM8HEADER}) \[GC \(%{DATA:[elasticsearch][gc][phase][name]}\) \[%{BASE10NUM} CMS-initial-mark: %{BASE10NUM:[elasticsearch][gc][old_gen][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][old_gen][size_kb]:int}K\)\] %{BASE10NUM:[elasticsearch][gc][heap][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][heap][size_kb]:int}K\), %{BASE10NUM:[elasticsearch][gc][phase][duration_sec]:float} secs\] %{PROCTIME}",
"%{JVM9HEADER} GC\(%{BASE10NUM}\) ParNew: %{BASE10NUM}K-\>%{BASE10NUM:[elasticsearch][gc][young_gen][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][young_gen][size_kb]:int}K\)",
"%{JVM9HEADER} GC\(%{BASE10NUM}\) Old: %{BASE10NUM}K-\>%{BASE10NUM:[elasticsearch][gc][old_gen][used_kb]:int}K\(%{BASE10NUM:[elasticsearch][gc][old_gen][size_kb]:int}K\)",
"(?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"JVM8HEADER" => "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:[elasticsearch][gc][jvm_runtime_sec]:float}:"
"JVM9HEADER" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{POSINT:[process][pid]:int}\]\[%{DATA:[elasticsearch][gc][tags]}%{SPACE}*\]"
"PROCTIME" => "\[Times: user=%{BASE10NUM:[elasticsearch][gc][phase][cpu_time][user_sec]:float} sys=%{BASE10NUM:[elasticsearch][gc][phase][cpu_time][sys_sec]:float}, real=%{BASE10NUM:[elasticsearch][gc][phase][cpu_time][real_sec]:float} secs\]"
}
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [event][timezone] {
date {
match => [ "timestamp", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "timestamp"
}
} else {
date {
match => [ "timestamp", "ISO8601" ]
remove_field => "timestamp"
}
}
if [elasticsearch][gc][tags] {
mutate {
split => { "[elasticsearch][gc][tags]" => "," }
}
}
}
} else if [fileset][name] == "server" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[elasticsearch][server]"
}
if [elasticsearch][server][type] != "server" {
drop { }
}
de_dot {
fields => [ "[elasticsearch][server][cluster.name]", "[elasticsearch][server][node.name]", "[elasticsearch][server][cluster.uuid]", "[elasticsearch][server][node.id]" ]
nested => true
}
mutate {
remove_field => [ "[elasticsearch][slowlog][type]" ]
rename => {
"[elasticsearch][server][level]" => "[log][level]"
"[elasticsearch][server][component]" => "[elasticsearch][component]"
"[elasticsearch][server][cluster][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][server][node][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][server][cluster][uuid]" => "[elasticsearch][cluster][uuid]"
"[elasticsearch][server][node][id]" => "[elasticsearch][node][id]"
}
}
grok {
match => { "[elasticsearch][server][message]" => [
"%{GC_ALL}",
"%{GC_YOUNG}",
"((\[%{INDEXNAME:[elasticsearch][index][name]}\]|\[%{INDEXNAME:[elasticsearch][index][name]}\/%{DATA:[elasticsearch][index][id]}\]))?%{SPACE}%{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"INDEXNAME" => "[a-zA-Z0-9_.-]*"
"GC_ALL" => "\[gc\]\[%{NUMBER:[elasticsearch][server][gc][overhead_seq]:int}\] overhead, spent \[%{NUMBER:[elasticsearch][server][gc][collection_duration][time]:float}%{DATA:[elasticsearch][server][gc][collection_duration][unit]}\] collecting in the last \[%{NUMBER:[elasticsearch][server][gc][observation_duration][time]:float}%{DATA:[elasticsearch][server][gc][observation_duration][unit]}\]"
"GC_YOUNG" => "\[gc\]\[young\]\[%{NUMBER:[elasticsearch][server][gc][young][one]:int}\]\[%{NUMBER:[elasticsearch][server][gc][young][two]}\]%{SPACE}%{GREEDYMULTILINE:message}"
}
remove_field => [ "[elasticsearch][server][message]" ]
}
} else {
grok {
match => { "message" => [
"%{LOG_HEADER}%{GC_ALL}",
"%{LOG_HEADER}%{GC_YOUNG}",
"%{LOG_HEADER}%{SPACE}((\[%{INDEXNAME:[elasticsearch][index][name]}\]|\[%{INDEXNAME:[elasticsearch][index][name]}\/%{DATA:[elasticsearch][index][id]}\]))?%{SPACE}%{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"INDEXNAME" => "[a-zA-Z0-9_.-]*"
"GC_ALL" => "\[gc\]\[%{NUMBER:[elasticsearch][server][gc][overhead_seq]:int}\] overhead, spent \[%{NUMBER:[elasticsearch][server][gc][collection_duration][time]:float}%{DATA:[elasticsearch][server][gc][collection_duration][unit]}\] collecting in the last \[%{NUMBER:[elasticsearch][server][gc][observation_duration][time]:float}%{DATA:[elasticsearch][server][gc][observation_duration][unit]}\]"
"GC_YOUNG" => "\[gc\]\[young\]\[%{NUMBER:[elasticsearch][server][gc][young][one]:int}\]\[%{NUMBER:[elasticsearch][server][gc][young][two]:int}\]%{SPACE}%{GREEDYMULTILINE:message}"
"LOG_HEADER" => "\[%{TIMESTAMP_ISO8601:[elasticsearch][server][timestamp]}\]\[%{LOGLEVEL:[log][level]}%{SPACE}?\]\[%{DATA:[elasticsearch][server][component]}%{SPACE}\](%{SPACE})?(\[%{DATA:[elasticsearch][node][name]}\])?(%{SPACE})?"
}
}
}
if [event][timezone] {
date {
match => [ "[elasticsearch][server][timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[elasticsearch][server][timestamp]"
}
} else {
date {
match => [ "[elasticsearch][server][timestamp]", "ISO8601" ]
remove_field => "[elasticsearch][server][timestamp]"
}
}
mutate {
remove_field => [ "first_char" ]
}
}
} else if [fileset][name] == "slowlog" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[elasticsearch][slowlog]"
}
if [elasticsearch][slowlog][type] != "index_indexing_slowlog" and [elasticsearch][slowlog][type] != "index_search_slowlog" {
drop { }
}
de_dot {
fields => [ "[elasticsearch][slowlog][cluster.name]", "[elasticsearch][slowlog][node.name]", "[elasticsearch][slowlog][cluster.uuid]", "[elasticsearch][slowlog][node.id]" ]
nested => true
}
mutate {
remove_field => [ "[elasticsearch][slowlog][type]" ]
rename => {
"[elasticsearch][slowlog][level]" => "[log][level]"
"[elasticsearch][slowlog][cluster][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][slowlog][node][name]" => "[elasticsearch][cluster][name]"
"[elasticsearch][slowlog][cluster][uuid]" => "[elasticsearch][cluster][uuid]"
"[elasticsearch][slowlog][node][id]" => "[elasticsearch][node][id]"
}
}
grok {
match => { "[elasticsearch][slowlog][message]" => [ "(\[%{INDEXNAME:[elasticsearch][index][name]}\]\[%{NUMBER:[elasticsearch][shard][id]:int}\])?(%{SPACE})?(\[%{INDEXNAME:[elasticsearch][index][name]}\/%{DATA:[elasticsearch][index][id]}\])?(%{SPACE})?%{SPACE}(took\[%{DATA:[elasticsearch][slowlog][took]}\],)?%{SPACE}(took_millis\[%{NUMBER:[elasticsearch][slowlog][duration]:int}\],)?%{SPACE}(type\[%{DATA:[elasticsearch][slowlog][type]}\],)?%{SPACE}(id\[%{DATA:[elasticsearch][slowlog][id]}\],)?%{SPACE}(routing\[%{DATA:[elasticsearch][slowlog][routing]}\],)?%{SPACE}(total_hits\[%{NUMBER:[elasticsearch][slowlog][total_hits]:int}\],)?%{SPACE}(types\[%{DATA:[elasticsearch][slowlog][types]}\],)?%{SPACE}(stats\[%{DATA:[elasticsearch][slowlog][stats]}\],)?%{SPACE}(search_type\[%{DATA:[elasticsearch][slowlog][search_type]}\],)?%{SPACE}(total_shards\[%{NUMBER:[elasticsearch][slowlog][total_shards]:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:[elasticsearch][slowlog][source_query]}\])?,?%{SPACE}(extra_source\[%{DATA:[elasticsearch][slowlog][extra_source]}\])?,?" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"INDEXNAME" => "[a-zA-Z0-9_.-]*"
}
remove_field => [ "[elasticsearch][slowlog][message]" ]
}
} else {
grok {
match => { "message" => [ "\[%{TIMESTAMP_ISO8601:[elasticsearch][slowlog][timestamp]}\]\[%{WORD:[log][level]}(%{SPACE})?\]\[%{DATA:[elasticsearch][slowlog][logger]}\]%{SPACE}\[%{WORD:[elasticsearch][node][name]}\](%{SPACE})?(\[%{INDEXNAME:[elasticsearch][index][name]}\]\[%{NUMBER:[elasticsearch][shard][id]:int}\])?(%{SPACE})?(\[%{INDEXNAME:[elasticsearch][index][name]}\/%{DATA:[elasticsearch][index][id]}\])?(%{SPACE})?%{SPACE}(took\[%{DATA:[elasticsearch][slowlog][took]}\],)?%{SPACE}(took_millis\[%{NUMBER:[elasticsearch][slowlog][duration]:int}\],)?%{SPACE}(type\[%{DATA:[elasticsearch][slowlog][type]}\],)?%{SPACE}(id\[%{DATA:[elasticsearch][slowlog][id]}\],)?%{SPACE}(routing\[%{DATA:[elasticsearch][slowlog][routing]}\],)?%{SPACE}(total_hits\[%{NUMBER:[elasticsearch][slowlog][total_hits]:int}\],)?%{SPACE}(types\[%{DATA:[elasticsearch][slowlog][types]}\],)?%{SPACE}(stats\[%{DATA:[elasticsearch][slowlog][stats]}\],)?%{SPACE}(search_type\[%{DATA:[elasticsearch][slowlog][search_type]}\],)?%{SPACE}(total_shards\[%{NUMBER:[elasticsearch][slowlog][total_shards]:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:[elasticsearch][slowlog][source_query]}\])?,?%{SPACE}(extra_source\[%{DATA:[elasticsearch][slowlog][extra_source]}\])?,?" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"INDEXNAME" => "[a-zA-Z0-9_.-]*"
}
}
if ("_grokparsefailure" not in [tags]) {
if [elasticsearch][slowlog][stats] {
mutate {
split => { "[elasticsearch][slowlog][stats]" => "," }
}
}
}
}
if [elasticsearch][slowlog][timestamp] {
if [event][timezone] {
date {
match => [ "[elasticsearch][slowlog][timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[elasticsearch][slowlog][timestamp]"
}
} else {
date {
match => [ "[elasticsearch][slowlog][timestamp]", "ISO8601" ]
remove_field => "[elasticsearch][slowlog][timestamp]"
}
}
}
if [elasticsearch][slowlog][duration] {
math {
calculate => [
[ "multiply", "[elasticsearch][slowlog][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
remove_field => [ "[elasticsearch][slowlog][duration]" ]
}
}
mutate {
remove_field => [ "first_char" ]
}
}
}
} else if [event][module] == "haproxy" {
if [fileset][name] == "log" {
grok {
match => { "message" => [
"%{HAPROXY_DATE:[haproxy][request_date]} %{IPORHOST:[haproxy][source]} %{PROG:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYDATA} %{IPORHOST:[source][address]}:%{POSINT:[source][port]:int} %{WORD} %{IPORHOST:[destination][address]}:%{POSINT:[destination][port]:int} \(%{WORD:[haproxy][frontend_name]}/%{WORD:[haproxy][mode]}\)",
"(%{NOTSPACE:[process][name]}\[%{NUMBER:[process][pid]:int}\]: )?%{IP:[source][address]}:%{NUMBER:[source][port]:int} \[%{NOTSPACE:[haproxy][request_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{NUMBER:[haproxy][http][request][time_wait_ms]:int}/%{NUMBER:[haproxy][total_waiting_time_ms]:int}/%{NUMBER:[haproxy][connection_wait_time_ms]:int}/%{NUMBER:[haproxy][http][request][time_wait_without_data_ms]:int}/%{NUMBER:[temp][duration]:int} %{NUMBER:[haproxy][http][response][status_code]:int} %{NUMBER:[haproxy][bytes_read]:int} %{NOTSPACE:[haproxy][http][request][captured_cookie]} %{NOTSPACE:[haproxy][http][response][captured_cookie]} %{NOTSPACE:[haproxy][termination_state]} %{NUMBER:[haproxy][connections][active]:int}/%{NUMBER:[haproxy][connections][frontend]:int}/%{NUMBER:[haproxy][connections][backend]:int}/%{NUMBER:[haproxy][connections][server]:int}/%{NUMBER:[haproxy][connections][retries]:int} %{NUMBER:[haproxy][server_queue]:int}/%{NUMBER:[haproxy][backend_queue]:int} (\{%{DATA:[haproxy][http][request][captured_headers]}\} \{%{DATA:[haproxy][http][response][captured_headers]}\} |\{%{DATA}\} )?\"%{GREEDYDATA:[haproxy][http][request][raw_request_line]}\"",
"(%{NOTSPACE:[process][name]}\[%{NUMBER:[process][pid]:int}\]: )?%{IP:[source][address]}:%{NUMBER:[source][port]:int} \[%{NOTSPACE:[haproxy][request_date]}\] %{NOTSPACE:[haproxy][frontend_name]}/%{NOTSPACE:[haproxy][bind_name]} %{GREEDYDATA:[haproxy][error_message]}",
"%{HAPROXY_DATE} %{IPORHOST:[haproxy][source]} (%{NOTSPACE:[process][name]}\[%{NUMBER:[process][pid]:int}\]: )?%{IP:[source][address]}:%{NUMBER:[source][port]:int} \[%{NOTSPACE:[haproxy][request_date]}\] %{NOTSPACE:[haproxy][frontend_name]} %{NOTSPACE:[haproxy][backend_name]}/%{NOTSPACE:[haproxy][server_name]} %{NUMBER:[haproxy][total_waiting_time_ms]:int}/%{NUMBER:[haproxy][connection_wait_time_ms]:int}/%{NUMBER:[temp][duration]:int} %{NUMBER:[haproxy][bytes_read]:int} %{NOTSPACE:[haproxy][termination_state]} %{NUMBER:[haproxy][connections][active]:int}/%{NUMBER:[haproxy][connections][frontend]:int}/%{NUMBER:[haproxy][connections][backend]:int}/%{NUMBER:[haproxy][connections][server]:int}/%{NUMBER:[haproxy][connections][retries]:int} %{NUMBER:[haproxy][server_queue]:int}/%{NUMBER:[haproxy][backend_queue]:int}" ]
}
pattern_definitions => {
"HAPROXY_DATE" => "(%{MONTHDAY}[/-]%{MONTH}[/-]%{YEAR}:%{HOUR}:%{MINUTE}:%{SECOND})|%{SYSLOGTIMESTAMP}"
}
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[haproxy][request_date]", "dd/MMM/yyyy:HH:mm:ss.SSS", "MMM dd HH:mm:ss" ]
remove_field => [ "[haproxy][request_date]", "message" ]
}
if [source][address] {
grok {
match => { "[source][address]" => [ "^%{IP:[source][ip]}$" ] }
}
}
if [haproxy][http][request][captured_headers] {
mutate {
split => { "[haproxy][http][request][captured_headers]" => "\|" }
}
}
if [haproxy][http][response][captured_headers] {
mutate {
split => { "[haproxy][http][response][captured_headers]" => "\|" }
}
}
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
remove_field => [ "[temp][duration]" ]
}
}
if [haproxy][bytes_read] {
if [haproxy][http] {
mutate {
copy => { "[haproxy][bytes_read]" => "[http][response][bytes]" }
}
}
}
}
}
} else if [event][module] == "icinga" {
if [fileset][name] == "debug" {
grok {
match => { "message" => [ "\[%{TIMESTAMP:[icinga][debug][timestamp]}\] %{WORD:[log][level]}/%{WORD:[icinga][debug][facility]}: %{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"TIMESTAMP" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}"
"GREEDYMULTILINE" => "(.|\n)*"
}
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[icinga][debug][timestamp]", "yyyy-MM-dd HH:mm:ss Z" ]
remove_field => "[icinga][debug][timestamp]"
}
}
} else if [fileset][name] == "main" {
grok {
match => { "message" => [ "\[%{TIMESTAMP:[icinga][main][timestamp]}\] %{WORD:[log][level]}/%{WORD:[icinga][main][facility]}: %{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"TIMESTAMP" => "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}"
"GREEDYMULTILINE" => "(.|\n)*"
}
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[icinga][main][timestamp]", "yyyy-MM-dd HH:mm:ss Z" ]
remove_field => "[icinga][main][timestamp]"
}
}
} else if [fileset][name] == "startup" {
grok {
match => { "message" => [ "%{WORD:[log][level]}/%{WORD:[icinga][startup][facility]}: %{GREEDYMULTILINE:message}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
}
}
}
} else if [event][module] == "iis" {
if [fileset][name] == "access" {
grok {
match => { "message" => [
"%{TIMESTAMP_ISO8601:[iis][access][time]} %{IPORHOST:[destination][address]} %{WORD:[http][request][method]} %{URIPATHWITHBRACKET:[url][path]} %{NOTSPACE:[url][query]} %{NUMBER:[destination][port]:int} %{NOTSPACE:[user][name]} %{IPORHOST:[source][address]} %{NOTSPACE:[user_agent][original]} %{NOTSPACE:[http][request][referrer]} %{NUMBER:[http][response][status_code]:int} %{NUMBER:[iis][access][sub_status]:int} %{NUMBER:[iis][access][win32_status]:int} %{NUMBER:[temp][duration]:int}",
"%{TIMESTAMP_ISO8601:[iis][access][time]} %{NOTSPACE:[iis][access][site_name]} %{WORD:[http][request][method]} %{URIPATH:[url][path]} %{NOTSPACE:[url][query]} %{NUMBER:[destination][port]:int} %{NOTSPACE:[user][name]} %{IPORHOST:[source][address]} %{NOTSPACE:[user_agent][original]} %{NOTSPACE:[iis][access][cookie]} %{NOTSPACE:[http][request][referrer]} %{NOTSPACE:[destination][domain]} %{NUMBER:[http][response][status_code]:int} %{NUMBER:[iis][access][sub_status]:int} %{NUMBER:[iis][access][win32_status]:int} %{NUMBER:[http][response][body][bytes]:int} %{NUMBER:[http][request][body][bytes]:int} %{NUMBER:[temp][duration]:int}",
"%{TIMESTAMP_ISO8601:[iis][access][time]} %{NOTSPACE:[iis][access][site_name]} %{NOTSPACE:[iis][access][server_name]} %{IPORHOST:[destination][address]} %{WORD:[http][request][method]} %{URIPATH:[url][path]} %{NOTSPACE:[url][query]} %{NUMBER:[destination][port]:int} %{NOTSPACE:[user][name]} %{IPORHOST:[source][address]} HTTP/%{NUMBER:[http][version]} %{NOTSPACE:[user_agent][original]} %{NOTSPACE:[iis][access][cookie]} %{NOTSPACE:[http][request][referrer]} %{NOTSPACE:[destination][domain]} %{NUMBER:[http][response][status_code]:int} %{NUMBER:[iis][access][sub_status]:int} %{NUMBER:[iis][access][win32_status]:int} %{NUMBER:[http][response][body][bytes]:int} %{NUMBER:[http][request][body][bytes]:int} %{NUMBER:[temp][duration]:int}",
"%{TIMESTAMP_ISO8601:[iis][access][time]} \[%{IPORHOST:[destination][address]}\]\(http://%{IPORHOST:[destination][address]}\) %{WORD:[http][request][method]} %{URIPATH:[url][path]} %{NOTSPACE:[url][query]} %{NUMBER:[destination][port]:int} %{NOTSPACE:[user][name]} \[%{IPORHOST:[source][address]}\]\(http://%{IPORHOST:[source][address]}\) %{NOTSPACE:[user_agent][original]} %{NUMBER:[http][response][status_code]:int} %{NUMBER:[iis][access][sub_status]:int} %{NUMBER:[iis][access][win32_status]:int} %{NUMBER:[temp][duration]:int}",
"%{TIMESTAMP_ISO8601:[iis][access][time]} %{IPORHOST:[destination][address]} %{WORD:[http][request][method]} %{URIPATH:[url][path]} %{NOTSPACE:[url][query]} %{NUMBER:[destination][port]:int} %{NOTSPACE:[user][name]} %{IPORHOST:[source][address]} %{NOTSPACE:[user_agent][original]} %{NUMBER:[http][response][status_code]:int} %{NUMBER:[iis][access][sub_status]:int} %{NUMBER:[iis][access][win32_status]:int} %{NUMBER:[temp][duration]:int}" ]
}
pattern_definitions => {
"URIPATHWITHBRACKET" => "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-\[\]]*)+"
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
remove_field => [ "message" ]
add_field => { "[event][created]" => "%{@timestamp}" }
}
if [event][timezone] {
date {
match => [ "[iis][access][time]", "yyyy-MM-dd HH:mm:ss" ]
timezone => "%{[event][timezone]}"
remove_field => "[iis][access][time]"
}
} else {
date {
match => [ "[iis][access][time]", "yyyy-MM-dd HH:mm:ss" ]
remove_field => "[iis][access][time]"
}
}
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
remove_field => [ "[temp][duration]" ]
}
}
if [user_agent][original] {
useragent {
source => "[user_agent][original]"
}
}
if [destination][address] {
grok {
match => { "[destination][address]" => [ "%{NOZONEIP:[destination][ip]}" ] }
pattern_definitions => { "NOZONEIP" => "[^%]*" }
}
}
if [source][address] {
grok {
match => { "[source][address]" => [ "%{NOZONEIP:[source][ip]}" ] }
pattern_definitions => { "NOZONEIP" => "[^%]*" }
}
}
}
} else if [fileset][name] == "error" {
grok {
match => { "message" => [ "%{TIMESTAMP_ISO8601:[iis][error][time]} %{IPORHOST:[source][address]} %{NUMBER:[source][port]:int} %{IPORHOST:[destination][address]} %{IPORHOST:[destination][port]:int} (?:HTTP/%{NUMBER:[http][version]:float}|-) (?:%{WORD:[http][request][method]}|-) (?:%{URIPATHPARAM:[url][original]}|-)(?: -)? (?:%{NUMBER:[http][response][status_code]:int}|-) (?:%{NUMBER}|-) (?:%{NOTSPACE:[iis][error][reason_phrase]}|-) (?:%{NOTSPACE:[iis][error][queue_name]}|-)" ]
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
remove_field => [ "message" ]
add_field => { "[event][created]" => "%{@timestamp}" }
}
if [event][timezone] {
date {
match => [ "[iis][error][time]", "yyyy-MM-dd HH:mm:ss" ]
timezone => "%{[event][timezone]}"
remove_field => "[iis][error][time]"
}
} else {
date {
match => [ "[iis][error][time]", "yyyy-MM-dd HH:mm:ss" ]
remove_field => "[iis][error][time]"
}
}
if [destination][address] {
grok {
match => { "[destination][address]" => [ "%{NOZONEIP:[destination][ip]}" ] }
pattern_definitions => { "NOZONEIP" => "[^%]*" }
}
}
if [source][address] {
grok {
match => { "[source][address]" => [ "%{NOZONEIP:[source][ip]}" ] }
pattern_definitions => { "NOZONEIP" => "[^%]*" }
}
}
}
}
} else if [event][module] == "kafka" {
if [fileset][name] == "log" {
grok {
match => { "message" => [ "(?m)%{TIMESTAMP_ISO8601:[kafka][log][timestamp]}. %{LOGLEVEL:[log][level]} +%{JAVALOGMESSAGE:message} \(%{JAVACLASS:[kafka][log][class]}\)$[ \n]*(?'[kafka][log][trace][full]'.*)" ] }
}
if ("_grokparsefailure" not in [tags]) {
grok {
match => { "message" => [ "\[%{KAFKA_COMPONENT:[kafka][log][component]}\][,:.]? +%{JAVALOGMESSAGE:message}" ] }
pattern_definitions => { "KAFKA_COMPONENT" => "[^\]]*" }
tag_on_failure => [ "_grokparsefailure_kafka_log_component" ]
}
if ("_grokparsefailure_kafka_log_component" in [tags]) {
mutate {
add_field => { "[kafka][log][component]" => "unknown" }
remove_tag => [ "_grokparsefailure_kafka_log_component" ]
}
}
grok {
match => { "[kafka][log][trace][full]" => [ "%{JAVACLASS:[kafka][log][trace][class]}:\s*%{JAVALOGMESSAGE:[kafka][log][trace][message]}" ] }
tag_on_failure => [ "_grokparsefailure_kafka_log_trace" ]
}
if ("_grokparsefailure_kafka_log_trace" in [tags]) {
mutate {
remove_field => [ "[kafka][log][trace]" ]
remove_tag => [ "_grokparsefailure_kafka_log_trace" ]
}
} else {
mutate {
remove_field => [ "[kafka][log][trace][full]" ]
}
}
mutate {
add_field => { "[event][created]" => "%{@timestamp}" }
}
if [event][timezone] {
date {
match => [ "[kafka][log][timestamp]", "yyyy-MM-dd HH:mm:ss,SSS" ]
timezone => "%{[event][timezone]}"
remove_field => "[kafka][log][timestamp]"
}
} else {
date {
match => [ "[kafka][log][timestamp]", "yyyy-MM-dd HH:mm:ss,SSS" ]
remove_field => "[kafka][log][timestamp]"
}
}
}
}
} else if [event][module] == "kibana" {
if [fileset][name] == "log" {
mutate {
add_field => {
"[event][created]" => "%{@timestamp}"
"[service][name]" => "kibana"
}
}
if [json] {
mutate {
copy => { "json" => "[kibana][log][meta]" }
remove_field => "[json]"
}
}
if [kibana][log][meta][@timestamp] {
date {
match => [ "[kibana][log][meta][@timestamp]", "ISO8601" ]
remove_field => [ "[kibana][log][meta][@timestamp]" ]
}
}
mutate {
rename => {
"[kibana][log][meta][message]" => "message"
"[kibana][log][meta][state]" => "[kibana][log][state]"
"[kibana][log][meta][pid]" => "[process][pid]"
"[kibana][log][meta][tags]" => "[kibana][log][tags]"
"[kibana][log][meta][res][statusCode]" => "[http][response][status_code]"
"[kibana][log][meta][res][responseTime]" => "[temp][duration]"
"[kibana][log][meta][res][contentLength]" => "[http][response][body][bytes]"
"[kibana][log][meta][req][method]" => "[http][request][method]"
"[kibana][log][meta][req][headers][referer]" => "[http][request][referrer]"
"[kibana][log][meta][req][headers][user-agent]" => "[user_agent][original]"
"[kibana][log][meta][req][remoteAddress]" => "[source][address]"
"[kibana][log][meta][req][url]" => "[url][original]"
}
convert => {
"[process][pid]" => "integer"
"[http][response][status_code]" => "integer"
"[temp][duration]" => "integer"
"[http][response][content_length]" => "integer"
}
remove_field => [ "[kibana][log][meta][req][referer]", "[kibana][log][meta][statusCode]", "[kibana][log][meta][method]" ]
}
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
remove_field => [ "[temp][duration]" ]
}
}
if [user_agent][original] {
useragent {
source => "[user_agent][original]"
}
}
if [source][address] {
mutate {
add_field => { "[source][ip]" => "%{[source][address]}" }
}
}
}
} else if [event][module] == "logstash" {
if [fileset][name] == "log" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[logstash][log]"
remove_field => [ "message" ]
}
if [logstash][log][timeMillis] {
date {
match => [ "[logstash][log][timeMillis]", "UNIX_MS" ]
remove_field => "[logstash][log][timeMillis]"
}
}
mutate {
rename => {
"[logstash][log][loggerName]" => "[logstash][log][module]"
"[logstash][log][logEvent][message]" => "message"
"[logstash][log][logEvent]" => "[logstash][log][log_event]"
"[logstash][log][level]" => "[log][level]"
}
}
} else {
grok {
match => { "message" => [ "\[%{TIMESTAMP_ISO8601:[logstash][log][timestamp]}\]\[%{LOGSTASH_LOGLEVEL:[log][level]}\s?\]\[%{LOGSTASH_CLASS_MODULE:[logstash][log][module]}\s*\] %{GREEDYMULTILINE:message}" ] }
pattern_definitions => {
"LOGSTASH_CLASS_MODULE" => "[\w\.]+"
"LOGSTASH_LOGLEVEL" => "INFO|ERROR|DEBUG|FATAL|WARN|TRACE"
"GREEDYMULTILINE" => "(.|\n)*"
}
}
if ("_grokparsefailure" not in [tags]) {
if [event][timezone] {
date {
match => [ "[logstash][log][timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[logstash][log][timestamp]"
}
} else {
date {
match => [ "[logstash][log][timestamp]", "ISO8601" ]
remove_field => "[logstash][log][timestamp]"
}
}
}
}
mutate {
remove_field => [ "first_char" ]
}
}
} else if [fileset][name] == "slowlog" {
grok {
match => { "message" => [ "^%{CHAR:first_char}" ] }
pattern_definitions => { "CHAR" => "." }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [first_char] == "{" {
json {
source => "message"
target => "[logstash][slowlog]"
remove_field => "message"
}
if [logstash][slowlog][timeMillis] {
date {
match => [ "[logstash][slowlog][timeMillis]", "UNIX_MS" ]
remove_field => "[logstash][slowlog][timeMillis]"
}
}
mutate {
rename => {
"[logstash][slowlog][loggerName]" => "[logstash][slowlog][module]"
"[logstash][slowlog][logEvent][took_in_millis]" => "[logstash][slowlog][took_in_millis]"
"[logstash][slowlog][logEvent][took_in_nanos]" => "[logstash][slowlog][took_in_nanos]"
"[logstash][slowlog][logEvent][plugin_params]" => "[logstash][slowlog][plugin_params_object]"
"[logstash][slowlog][level]" => "[log][level]"
}
copy => { "[logstash][slowlog][logEvent][event]" => "[logstash][slowlog][event]" }
convert => {
"[logstash][slowlog][took_in_millis]" => "integer"
"[logstash][slowlog][took_in_nanos]" => "integer"
}
remove_field => [ "message", "[logstash][slowlog][logEvent][event]", "[logstash][slowlog][logEvent]" ]
}
if [logstash][slowlog][module] {
grok {
match => { "[logstash][slowlog][module]" => [ "slowlog.logstash.%{WORD:[logstash][slowlog][plugin_type]}.%{WORD:[logstash][slowlog][plugin_name]}" ] }
}
}
} else {
grok {
match => { "message" => [ "\[%{TIMESTAMP_ISO8601:[logstash][slowlog][timestamp]}\]\[%{LOGSTASH_LOGLEVEL:[log][level]}\s?\]\[%{LOGSTASH_CLASS_MODULE:[logstash][slowlog][module]}\] %{GREEDYDATA:message}" ] }
pattern_definitions => {
"LOGSTASH_CLASS_MODULE" => "[\w\.]+\s*"
"LOGSTASH_LOGLEVEL" => "INFO|ERROR|DEBUG|FATAL|WARN|TRACE"
}
}
if ("_grokparsefailure" not in [tags]) {
if [logstash][slowlog][module] {
grok {
match => { "[logstash][slowlog][module]" => [ "slowlog.logstash.%{WORD:[logstash][slowlog][plugin_type]}.%{WORD:[logstash][slowlog][plugin_name]}" ] }
}
}
grok {
match => { "message" => [ "{:plugin_params=>%{GREEDYDATA:[logstash][slowlog][plugin_params]}, :took_in_nanos=>%{NUMBER:[logstash][slowlog][took_in_nanos]:int}, :took_in_millis=>%{NUMBER:[logstash][slowlog][took_in_millis]:int}, :event=>%{GREEDYDATA:[logstash][slowlog][event]}}" ]
}
remove_field => "message"
}
if [event][timezone] {
date {
match => [ "[logstash][slowlog][timestamp]", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => "[logstash][slowlog][timestamp]"
}
} else {
date {
match => [ "[logstash][slowlog][timestamp]", "ISO8601" ]
remove_field => "[logstash][slowlog][timestamp]"
}
}
}
}
mutate {
remove_field => [ "first_char" ]
}
}
}
} else if [event][module] == "mongodb" {
if [fileset][name] == "log" {
grok {
match => { "message" => [ "%{TIMESTAMP_ISO8601:[mongodb][log][timestamp]}%{SPACE}%{MONGO3_SEVERITY:[log][level]}%{SPACE}%{MONGO3_COMPONENT:[mongodb][log][component]}%{SPACE}(?:\[%{DATA:[mongodb][log][context]}\])?%{SPACE}%{GREEDYDATA:message}" ] }
pattern_definitions => {
MONGO3_SEVERITY => "\w"
MONGO3_COMPONENT => "%{WORD}|-"
}
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[mongodb][log][timestamp]", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ]
remove_field => "[mongodb][log][timestamp]"
}
}
}
} else if [event][module] == "mysql" {
if [fileset][name] == "error" {
grok {
match => { "message" => [
"%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[log][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
"%{DATA:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]:int} \[%{DATA:[log][level]}\] %{GREEDYDATA:message}",
"%{GREEDYDATA:message}" ]
}
pattern_definitions => {
"LOCALDATETIME" => "[0-9]+ %{TIME}"
}
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [mysql][error][timestamp] {
date {
match => [ "[mysql][error][timestamp]", "ISO8601", "yyMMdd H:m:s" ]
remove_field => "[mysql][error][time]"
}
}
}
} else if [fileset][name] == "slowlog" {
grok {
match => { "message" => [ "^# User@Host: %{USER:[user][name]}(\[%{USER:[mysql][slowlog][current_user]}\])? @ %{HOSTNAME:[source][domain]}? \[%{IP:[source][ip]}?\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:[mysql][thread_id]:int}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:[mysql][thread_id]:int}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:[mysql][slowlog][schema]}?%{METRICSPACE})?(Last_errno: %{NUMBER:[mysql][slowlog][last_errno]:int}%{METRICSPACE})?(Killed: %{NUMBER:[mysql][slowlog][killed]:int}%{METRICSPACE})?(QC_hit: %{WORD:[mysql][slowlog][query_cache_hit]}%{METRICSPACE})?(Query_time: %{NUMBER:[temp][duration]:float}%{METRICSPACE})?(Lock_time: %{NUMBER:[mysql][slowlog][lock_time][sec]:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:[mysql][slowlog][rows_sent]:int}%{METRICSPACE})?(Rows_examined: %{NUMBER:[mysql][slowlog][rows_examined]:int}%{METRICSPACE})?(Rows_affected: %{NUMBER:[mysql][slowlog][rows_affected]:int}%{METRICSPACE})?(Thread_id: %{NUMBER:[mysql][thread_id]:int}%{METRICSPACE})?(Errno: %{NUMBER:[mysql][slowlog][last_errno]:int}%{METRICSPACE})?(Killed: %{NUMBER:[mysql][slowlog][killed]:int}%{METRICSPACE})?(Bytes_received: %{NUMBER:[mysql][slowlog][bytes_received]:int}%{METRICSPACE})?(Bytes_sent: %{NUMBER:[mysql][slowlog][bytes_sent]:int}%{METRICSPACE})?(Read_first: %{NUMBER:[mysql][slowlog][read_first]:int}%{METRICSPACE})?(Read_last: %{NUMBER:[mysql][slowlog][read_last]:int}%{METRICSPACE})?(Read_key: %{NUMBER:[mysql][slowlog][read_key]:int}%{METRICSPACE})?(Read_next: %{NUMBER:[mysql][slowlog][read_next]:int}%{METRICSPACE})?(Read_prev: %{NUMBER:[mysql][slowlog][read_prev]:int}%{METRICSPACE})?(Read_rnd: %{NUMBER:[mysql][slowlog][read_rnd]:int}%{METRICSPACE})?(Read_rnd_next: %{NUMBER:[mysql][slowlog][read_rnd_next]:int}%{METRICSPACE})?(Sort_merge_passes: %{NUMBER:[mysql][slowlog][sort_merge_passes]:int}%{METRICSPACE})?(Sort_range_count: %{NUMBER:[mysql][slowlog][sort_range_count]:int}%{METRICSPACE})?(Sort_rows: %{NUMBER:[mysql][slowlog][sort_rows]:int}%{METRICSPACE})?(Sort_scan_count: %{NUMBER:[mysql][slowlog][sort_scan_count]:int}%{METRICSPACE})?(Created_tmp_disk_tables: %{NUMBER:[mysql][slowlog][tmp_disk_tables]:int}%{METRICSPACE})?(Created_tmp_tables: %{NUMBER:[mysql][slowlog][tmp_tables]:int}%{METRICSPACE})?(Tmp_tables: %{NUMBER:[mysql][slowlog][tmp_tables]:int}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:[mysql][slowlog][tmp_disk_tables]:int}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:[mysql][slowlog][tmp_table_sizes]:int}%{METRICSPACE})?(Start: %{TIMESTAMP_ISO8601:[event][start]}%{METRICSPACE})?(End: %{TIMESTAMP_ISO8601:[event][end]}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:[mysql][slowlog][innodb][trx_id]}%{METRICSPACE})?(QC_Hit: %{WORD:[mysql][slowlog][query_cache_hit]}%{METRICSPACE})?(Full_scan: %{WORD:[mysql][slowlog][full_scan]}%{METRICSPACE})?(Full_join: %{WORD:[mysql][slowlog][full_join]}%{METRICSPACE})?(Tmp_table: %{WORD:[mysql][slowlog][tmp_table]}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:[mysql][slowlog][tmp_table_on_disk]}%{METRICSPACE})?(Filesort: %{WORD:[mysql][slowlog][filesort]}%{METRICSPACE})?(Filesort_on_disk: %{WORD:[mysql][slowlog][filesort_on_disk]}%{METRICSPACE})?(Merge_passes: %{NUMBER:[mysql][slowlog][merge_passes]:int}%{METRICSPACE})?(Priority_queue: %{WORD:[mysql][slowlog][priority_queue]}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:[mysql][slowlog][innodb][io_r_ops]:int}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:[mysql][slowlog][innodb][io_r_bytes]:int}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:[mysql][slowlog][innodb][io_r_wait][sec]:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:[mysql][slowlog][innodb][rec_lock_wait][sec]:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:[mysql][slowlog][innodb][queue_wait][sec]:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:[mysql][slowlog][innodb][pages_distinct]:int}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:[mysql][slowlog][log_slow_rate_type]}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:[mysql][slowlog][log_slow_rate_limit]:int}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:[mysql][slowlog][schema]};\n)?SET timestamp=%{NUMBER:[mysql][slowlog][timestamp]:int};\n%{GREEDYMULTILINE:[mysql][slowlog][query]}" ] }
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"METRICSPACE" => "([ #\n]*)"
"EXPLAIN" => "(# explain:.*\n|#\s*\n)*"
}
}
if ("_grokparsefailure" not in [tags]) {
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
}
mutate {
convert => { "[event][duration]" => "integer" }
remove_field => [ "[temp][duration]" ]
}
}
date {
match => [ "[mysql][slowlog][timestamp]", "UNIX" ]
remove_field => [ "[mysql][slowlog][timestamp]", "message" ]
}
}
}
} else if [event][module] == "nats" {
if [fileset][name] == "log" {
grok {
match => { "message" => [ "\[%{POSINT:[process][pid]:int}\]( %{NATSTIME:[nats][log][timestamp]})? \[%{NATSLOGLEVEL:[log][level]}\] %{GREEDYDATA:[nats][log][info]}" ] }
pattern_definitions => {
"NATSTIME" => "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}"
"NATSLOGLEVEL" => "(INF|DBG|WRN|ERR|FTL|TRC)"
}
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [nats][log][info] {
grok {
match => { "[nats][log][info]" => [
"%{IPV4:[client][ip]}:%{POSINT:[client][port]:int} - cid:%{POSINT:[nats][log][client][id]:int} - %{GREEDYDATA:[nats][log][msg][info]}",
"%{GREEDYDATA:[nats][log][msg][data]}" ]
}
}
if ("_grokparsefailure" not in [tags]) {
if [nats][log][msg][info] {
grok {
match => { "[nats][log][msg][info]" => [
"%{NATSDIRECTION:[network][direction]} %{NATSPAYLOAD:[nats][log][msg][type]}: \[%{GREEDYDATA:[nats][log][msg][payload]}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSNOINFO:[nats][log][msg][type]}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSUNSUB:[nats][log][msg][type]}\s+%{POSINT:[nats][log][msg][sid]:int}(\s+%{POSINT:[nats][log][msg][max_messages]:int})?\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSPUB:[nats][log][msg][type]}\s+%{NOTSPACE:[nats][log][msg][subject]}(\s+%{NOTSPACE:[nats][log][msg][reply_to]})?\s+%{POSINT:[nats][log][msg][bytes]:int}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSSUB:[nats][log][msg][type]}\s+%{NOTSPACE:[nats][log][msg][subject]}(\s+%{NOTSPACE:[nats][log][msg][queue_group]})?\s+%{POSINT:[nats][log][msg][sid]:int}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSMSG:[nats][log][msg][type]}\s+%{NOTSPACE:[nats][log][msg][subject]}\s+%{POSINT:[nats][log][msg][sid]:int}(\s+%{NOTSPACE:[nats][log][msg][reply_to]})?\s+%{POSINT:[nats][log][msg][bytes]:int}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSCONNECTION:[nats][log][msg][type]}\s+%{GREEDYDATA:[nats][log][msg][data]}\]",
"%{NATSDIRECTION:[network][direction]} \[%{NATSERROR:[nats][log][msg][type]}\s+%{GREEDYDATA:[nats][log][msg][error]}\]",
"%{GREEDYDATA:[nats][log][msg][data]}" ]
}
pattern_definitions => {
"NATSDIRECTION" => "(<<-|->>)"
"NATSMSG" => "MSG"
"NATSPUB" => "PUB"
"NATSSUB" => "SUB"
"NATSUNSUB" => "UNSUB"
"NATSPAYLOAD" => "MSG_PAYLOAD"
"NATSERROR" => "-ERROR"
"NATSPING" => "PING"
"NATSPONG" => "PONG"
"NATSOK" => "OK"
"NATSCONNECT" => "CONNECT"
"NATSINFO" => "INFO"
"NATSCONNECTION" => "(?:%{NATSCONNECT}|%{NATSINFO})"
"NATSNOINFO" => "(?:%{NATSPING}|%{NATSPONG}|%{NATSOK})"
}
remove_field => [ "[nats][log][info]", "[nats][log][msg][info]", "message" ]
}
if ("_grokparsefailure" not in [tags]) {
if [nats][log][msg][payload] {
mutate {
remove_field => "[nats][log][msg][payload]"
}
}
if [nats][log][msg][data] {
mutate {
add_field => { "message" => "%{[nats][log][msg][data]}" }
}
}
}
}
}
}
if [nats][log][timestamp] {
date {
match => [ "[nats][log][timestamp]", "yyyy/MM/dd HH:mm:ss.SSSSSS" ]
remove_field => "[nats][log][timestamp]"
}
}
}
}
} else if [event][module] == "nginx" {
if [fileset][name] == "access" {
grok {
match => { "message" => ["\"?(?:%{IP_LIST:[nginx][access][remote_ip_list]}|%{DATA:[source][address]}) - %{DATA:[user][name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{DATA:[nginx][access][info]}\" %{NUMBER:[http][response][status_code]:int} %{NUMBER:[http][response][body][bytes]:int} \"%{DATA:[http][request][referrer]}\" \"%{DATA:[user_agent][original]}\""] }
pattern_definitions => {
"IP_LIST" => "%{IP}(\"?,?\s*%{IP})*"
}
}
if ("_grokparsefailure" not in [tags]) {
grok {
match => { "[nginx][access][info]" => [
"%{WORD:[http][request][method]} %{DATA:[url][original]} HTTP/%{NUMBER:[http][version]:float}",
""]
}
remove_field => [ "[nginx][access][info]", "message" ]
add_field => { "[event][created]" => "%{@timestamp}" }
}
if [nginx][access][remote_ip_list] {
mutate {
split => { "[nginx][access][remote_ip_list]" => ", " }
}
if [nginx][access][remote_ip_list][0] {
mutate {
add_field => {
"[source][ip]" => "%{[nginx][access][remote_ip_list][0]}"
"[source][address]" => "%{[source][ip]}"
}
}
}
}
if [event][timezone] {
date {
match => [ "[nginx][access][time]", "dd/MMM/yyyy:H:m:s Z" ]
timezone => "%{[event][timezone]}"
remove_field => "[nginx][access][time]"
}
} else {
date {
match => [ "[nginx][access][time]", "dd/MMM/yyyy:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
}
if [user_agent][original] {
useragent {
source => "[user_agent][original]"
}
}
}
} else if [fileset][name] == "error" {
grok {
match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[log][level]}\] %{NUMBER:[process][pid]:int}#%{NUMBER:[process][thread][id]:int}: (\*%{NUMBER:[nginx][error][connection_id]:int} )?%{GREEDYDATA:message}"] }
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
if [event][timezone] {
date {
match => [ "[nginx][error][time]", "yyyy/MM/dd H:m:s" ]
timezone => "%{[event][timezone]}"
remove_field => "[nginx][error][time]"
}
} else {
date {
match => [ "[nginx][error][time]", "yyyy/MM/dd H:m:s" ]
remove_field => "[nginx][error][time]"
}
}
}
}
} else if [event][module] == "osquery" {
if [fileset][name] == "result" {
mutate {
add_field => { "[event][created]" => "%{@timestamp}" }
}
if [json][unixTime] {
date {
match => [ "[json][unixTime]", "UNIX" ]
}
}
if [json] {
mutate {
copy => { "json" => "[osquery][result]" }
remove_field => "[json]"
}
}
mutate {
rename => {
"[osquery][result][hostIdentifier]" => "[osquery][result][host_identifier]"
"[osquery][result][unixTime]" => "[osquery][result][unix_time]"
"[osquery][result][calendarTime]" => "[osquery][result][calendar_time]"
}
}
}
} else if [event][module] == "postgresql" {
if [fileset][name] == "log" {
grok {
match => { "message" => [ "^%{LOCALDATETIME:[postgresql][log][timestamp]} %{WORD:[event][timezone]} \[%{NUMBER:[process][pid]:int}(-%{BASE16FLOAT:[postgresql][log][core_id]:int})?\] ((\[%{USERNAME:[user][name]}\]@\[%{POSTGRESQL_DB_NAME:[postgresql][log][database]}\]|%{USERNAME:[user][name]}@%{POSTGRESQL_DB_NAME:[postgresql][log][database]}) )?%{WORD:[log][level]}: (duration: %{NUMBER:[temp][duration]:float} ms statement: %{GREEDYDATA:[postgresql][log][query]}|%{GREEDYDATA:message})" ]
}
pattern_definitions => {
"LOCALDATETIME" => "[-0-9]+ %{TIME}"
"GREEDYDATA" => "(.|\n|\t)*"
"POSTGRESQL_DB_NAME" => "[a-zA-Z0-9_]+[a-zA-Z0-9_\$]*"
}
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[postgresql][log][timestamp]", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
remove_field => "[postgresql][log][timestamp]"
}
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
}
mutate {
convert => { "[event][duration]" => "integer" }
remove_field => [ "[temp][duration]" ]
}
}
}
}
} else if [event][module] == "redis" {
if [fileset][name] == "log" {
grok {
match => { "message" => [
"(%{POSINT:[process][pid]:int}:%{CHAR:[redis][log][role]} )?(%{REDISTIMESTAMP1:[redis][log][timestamp]}||%{REDISTIMESTAMP2:[redis][log][timestamp]}) %{REDISLEVEL:[log][level]} %{GREEDYDATA:message}",
"%{POSINT:[process][pid]:int}:signal-handler \(%{POSINT:[redis][log][timestamp]}\) %{GREEDYDATA:message}" ]
}
pattern_definitions => {
"CHAR" => "[a-zA-Z]"
"REDISLEVEL" => "[.\-*#]"
"REDISTIMESTAMP1" => "%{MONTHDAY} %{MONTH} %{TIME}"
"REDISTIMESTAMP2" => "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}"
}
add_field => { "[event][created]" => "%{@timestamp}" }
}
if ("_grokparsefailure" not in [tags]) {
date {
match => [ "[redis][log][timestamp]", "dd MMM yyyy H:m:s.SSS", "dd MMM H:m:s.SSS", "dd MMM H:m:s", "UNIX" ]
remove_field => "[redis][log][timestamp]"
}
}
}
} else if [event][module] == "santa" {
if [fileset][name] == "log" {
grok{
match => { "message" => [
"\[%{TIMESTAMP_ISO8601:[process][start]}\] I santad: action=%{NOT_SEPARATOR:[santa][action]}\|decision=%{NOT_SEPARATOR:[santa][decision]}\|reason=%{NOT_SEPARATOR:[santa][reason]}\|sha256=%{NOT_SEPARATOR:[hash][sha256]}\|path=%{NOT_SEPARATOR:[process][executable]}(\|args=%{NOT_SEPARATOR:[process][args]})?(\|cert_sha256=%{NOT_SEPARATOR:[certificate][sha256]})?(\|cert_cn=%{NOT_SEPARATOR:[certificate][common_name]})?\|pid=%{NUMBER:[process][pid]:int}\|ppid=%{NUMBER:[process][ppid]:int}\|uid=%{NUMBER:[user][id]:int}\|user=%{NOT_SEPARATOR:[user][name]}\|gid=%{NUMBER:[group][id]:int}\|group=%{NOT_SEPARATOR:[group][name]}\|mode=%{WORD:[santa][mode]}",
"\[%{TIMESTAMP_ISO8601:timestamp}\] I santad: action=%{NOT_SEPARATOR:[santa][action]}\|mount=%{NOT_SEPARATOR:[santa][disk][mount]}\|volume=%{NOT_SEPARATOR:[santa][disk][volume]}\|bsdname=%{NOT_SEPARATOR:[santa][disk][bsdname]}\|fs=%{NOT_SEPARATOR:[santa][disk][fs]}\|model=%{NOT_SEPARATOR:[santa][disk][model]}\|serial=%{NOT_SEPARATOR:[santa][disk][serial]}\|bus=%{NOT_SEPARATOR:[santa][disk][bus]}\|dmgpath=%{NOT_SEPARATOR:[santa][disk][dmgpath]}?" ]
}
pattern_definitions => {
"NOT_SEPARATOR" => "[^\|]+"
}
}
if ("_grokparsefailure" not in [tags]) {
if [message] {
mutate {
rename => { "message" => "[log][original]" }
}
}
if [process][start] {
date {
match => [ "[process][start]", "ISO8601" ]
}
date {
match => [ "[process][start]", "ISO8601" ]
target => "[process][start]"
}
}
if [process][args] {
mutate {
split => { "[process][args]" => " " }
}
}
if [timestamp] {
date {
match => [ "timestamp", "ISO8601" ]
remove_field => "timestamp"
}
}
}
}
} else if [event][module] == "system" {
if [fileset][name] == "auth" {
grok {
match => { "message" => [
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[user][name]} from %{IPORHOST:[source][ip]} port %{NUMBER:[source][port]:int} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[user][name]} from %{IPORHOST:[source][ip]}",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: \s*%{DATA:[user][name]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: new group: name=%{DATA:[group][name]}, GID=%{NUMBER:[group][id]:int}",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: new user: name=%{DATA:[user][name]}, UID=%{NUMBER:[user][id]:int}, GID=%{NUMBER:[group][id]:int}, home=%{DATA:[system][auth][useradd][home]}, shell=%{DATA:[system][auth][useradd][shell]}$",
"%{TIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[host][hostname]}? %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYMULTILINE:[system][auth][message]}"]
}
pattern_definitions => {
"GREEDYMULTILINE"=> "(.|\n)*"
"TIMESTAMP" => "(?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})"
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
rename => { "[system][auth][message]" => "[message]" }
}
if [event][timezone] {
date {
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => [ "[system][auth][timestamp]" ]
}
} else {
date {
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
remove_field => [ "[system][auth][timestamp]" ]
}
}
}
} else if [fileset][name] == "syslog" {
grok {
match => { "message" => [
"%{TIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[host][hostname]} %{DATA:[process][name]}(?:\[%{POSINT:[process][pid]:int}\])?: %{GREEDYMULTILINE:[system][syslog][message]}",
"%{TIMESTAMP:[system][syslog][timestamp]} %{GREEDYMULTILINE:[system][syslog][message]}" ]
}
pattern_definitions => {
"GREEDYMULTILINE" => "(.|\n)*"
"TIMESTAMP" => "(?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})"
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
rename => { "[system][syslog][message]" => "[message]" }
}
if [event][timezone] {
date {
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
timezone => "%{[event][timezone]}"
remove_field => [ "[system][syslog][timestamp]" ]
}
} else {
date {
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
remove_field => [ "[system][syslog][timestamp]" ]
}
}
}
}
} else if [event][module] == "traefik" {
if [fileset][name] == "access" {
dissect {
mapping => {
"message" => '%{[source][address]} %{[traefik][access][user_identifier]} %{[user][name]} [%{[traefik][access][time]}] "%{[http][request][method]} %{[url][original]} HTTP/%{[http][version]}" %{[http][response][status_code]} %{[traefik][access][message]}'
}
}
if ("_dissectfailure" not in [tags]) {
grok {
match => { "[traefik][access][message]" => [ "(?:%{NUMBER:[http][response][body][bytes]:int}|-)( (?:\"%{DATA:[http][request][referrer]}\"|-)?( (?:\"%{DATA:[user_agent][original]}\"|-)?)?( (?:%{NUMBER:[traefik][access][request_count]:int}|-)?)?( (?:\"%{DATA:[traefik][access][frontend_name]}\"|-)?)?( \"%{DATA:[traefik][access][backend_url]}\")?( %{NUMBER:[temp][duration]:int}ms)?)?" ]
}
}
if ("_grokparsefailure" not in [tags]) {
mutate {
remove_field => [ "message", "[traefik][access][message]" ]
add_field => { "[event][created]" => "%{@timestamp}" }
convert => { "[http][response][status_code]" => "integer" }
}
date {
match => [ "[traefik][access][time]", "dd/MMM/yyyy:H:m:s Z" ]
remove_field => "[traefik][access][time]"
}
if [user_agent][original] {
useragent {
source => "[user_agent][original]"
}
}
}
if [temp][duration] {
math {
calculate => [
[ "multiply", "[temp][duration]", 1000000, "MEM[0]" ],
[ "round", "MEM[0]", 1, "[event][duration]" ]
]
remove_field => [ "[temp][duration]" ]
}
}
grok {
match => { "[source][address]" => [ "^(%{IP:[source][ip]}|%{HOSTNAME:[source][domain]})$" ] }
}
}
}
}
if [host][name] {
mutate {
rename => ["host", "beat" ]
add_field => { "host" => "%{[beat][name]}" }
}
}
if [@version] {
mutate {
remove_field => [ "@version" ]
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment