Skip to content

Instantly share code, notes, and snippets.

@dpmccabe
Created March 15, 2019 16:00
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dpmccabe/34dad1bd093182e72299c61f13a73c4e to your computer and use it in GitHub Desktop.
Save dpmccabe/34dad1bd093182e72299c61f13a73c4e to your computer and use it in GitHub Desktop.
Logstash setup
FROM logstash:6.6.1
ADD ./logstash.conf .
ADD ./logstash.yml .
ADD ./template.json .
ENV LS_JAVA_OPTS="-Dls.cgroup.cpuacct.path.override=/ -Dls.cgroup.cpu.path.override=/ -Djava.security.egd=file:/dev/urandom $LS_JAVA_OPTS"
CMD logstash -f ./logstash.conf --path.settings=. --verbose --log.level=debug
input {
tcp {
port => "${PORT}"
}
}
filter {
grok {
match => { "message" => "%{SYSLOG5424PRI:pri}%{NUMBER:rfc_version} %{TIMESTAMP_ISO8601:timestamp} %{USERNAME:app} %{USERNAME:dyno} %{USERNAME:proc} - Event %{USERNAME:event_id}: %{GREEDYDATA:message}" }
overwrite => ["message"]
remove_field => ["pri", "syslog5424_pri", "rfc_version", "dyno", "proc"]
}
if "_grokparsefailure" in [tags] {
drop { }
}
clone {
clones => ["for_es", "for_s3"]
}
if [type] == "for_es" {
json {
source => "message"
remove_field => ["message", "@version", "port", "host", "type", "@timestamp"]
}
mutate {
add_field => { "[@metadata][type]" => "for_es" }
}
} else if [type] == "for_s3" {
json {
source => "message"
}
mutate {
add_field => { "[@metadata][type]" => "for_s3" }
}
}
}
output {
stdout {
codec => rubydebug {
metadata => true
}
}
if [@metadata][type] == "for_es" {
elasticsearch {
hosts => "${ES_HOST}:${ES_PORT}"
user => "${ES_USER}"
password => "${ES_PWD}"
failure_type_logging_whitelist => ["document_already_exists_exception"]
index => "events"
document_type => "logs"
document_id => "%{[event_id]}"
template => "./template.json"
template_name => "events"
}
} else if [@metadata][type] == "for_s3" {
s3 {
access_key_id => "${AWS_ACCESS_KEY_ID}"
secret_access_key => "${AWS_SECRET_ACCESS_KEY}"
region => "${AWS_DEFAULT_REGION}"
bucket => "vpal-app-event-logs"
prefix => "%{[app]}/%{[deploy_env]}/%{+YYYY}/%{+MM}/%{+dd}"
rotation_strategy => "size_and_time"
time_file => 15
}
}
}
xpack.monitoring.enabled: false
xpack.management.enabled: false
2019-03-15T15:47:52.571853+00:00 app[web.1]: Could not find log4j2 configuration at path /usr/share/logstash/log4j2.properties. Using default config which logs errors to the console
2019-03-15T15:47:52.591118+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:52.588 [main] scaffold - Found module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
2019-03-15T15:47:52.596052+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:52.595 [main] registry - Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x71769c47 @directory="/usr/share/logstash/modules/fb_apache/configuration", @module_name="fb_apache", @kibana_version_parts=["6", "0", "0"]>}
2019-03-15T15:47:52.597751+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:52.597 [main] scaffold - Found module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
2019-03-15T15:47:52.598016+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:52.597 [main] registry - Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x6cb18bfb @directory="/usr/share/logstash/modules/netflow/configuration", @module_name="netflow", @kibana_version_parts=["6", "0", "0"]>}
2019-03-15T15:47:52.649800+00:00 app[web.1]: [INFO ] 2019-03-15 15:47:52.649 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
2019-03-15T15:47:52.654503+00:00 app[web.1]: [INFO ] 2019-03-15 15:47:52.654 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
2019-03-15T15:47:53.052049+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.051 [LogStash::Runner] runner - -------- Logstash Settings (* means modified) ---------
2019-03-15T15:47:53.052270+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - node.name: "dyno-af51488b-b39c-48b3-bd2a-858d725f1c5c"
2019-03-15T15:47:53.052351+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - *path.config: "./logstash.conf"
2019-03-15T15:47:53.052444+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - path.data: "/usr/share/logstash/data"
2019-03-15T15:47:53.052520+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - modules.cli: []
2019-03-15T15:47:53.052616+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - modules: []
2019-03-15T15:47:53.052679+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - modules_list: []
2019-03-15T15:47:53.052786+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - modules_variable_list: []
2019-03-15T15:47:53.052845+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - modules_setup: false
2019-03-15T15:47:53.052959+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - config.test_and_exit: false
2019-03-15T15:47:53.053024+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.052 [LogStash::Runner] runner - config.reload.automatic: false
2019-03-15T15:47:53.053120+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - config.reload.interval: 3000000000
2019-03-15T15:47:53.053201+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - config.support_escapes: false
2019-03-15T15:47:53.053280+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - config.field_reference.parser: "COMPAT"
2019-03-15T15:47:53.053356+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - metric.collect: true
2019-03-15T15:47:53.053880+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - pipeline.id: "main"
2019-03-15T15:47:53.053979+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - pipeline.system: false
2019-03-15T15:47:53.054073+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.053 [LogStash::Runner] runner - pipeline.workers: 2
2019-03-15T15:47:53.054147+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.output.workers: 1
2019-03-15T15:47:53.054572+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.batch.size: 125
2019-03-15T15:47:53.054661+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.batch.delay: 50
2019-03-15T15:47:53.054765+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.unsafe_shutdown: false
2019-03-15T15:47:53.054841+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.java_execution: false
2019-03-15T15:47:53.054936+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - pipeline.reloadable: true
2019-03-15T15:47:53.055003+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.054 [LogStash::Runner] runner - path.plugins: []
2019-03-15T15:47:53.055089+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - config.debug: false
2019-03-15T15:47:53.055166+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - *log.level: "debug" (default: "info")
2019-03-15T15:47:53.055256+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - version: false
2019-03-15T15:47:53.055342+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - help: false
2019-03-15T15:47:53.055411+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - log.format: "plain"
2019-03-15T15:47:53.055483+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - http.host: "127.0.0.1"
2019-03-15T15:47:53.055571+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - http.port: 9600..9700
2019-03-15T15:47:53.055642+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - http.environment: "production"
2019-03-15T15:47:53.055737+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - queue.type: "memory"
2019-03-15T15:47:53.055802+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - queue.drain: false
2019-03-15T15:47:53.055891+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - queue.page_capacity: 67108864
2019-03-15T15:47:53.055969+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - queue.max_bytes: 1073741824
2019-03-15T15:47:53.056050+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.055 [LogStash::Runner] runner - queue.max_events: 0
2019-03-15T15:47:53.056125+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - queue.checkpoint.acks: 1024
2019-03-15T15:47:53.056213+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - queue.checkpoint.writes: 1024
2019-03-15T15:47:53.056323+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - queue.checkpoint.interval: 1000
2019-03-15T15:47:53.056405+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - queue.checkpoint.retry: false
2019-03-15T15:47:53.056494+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - dead_letter_queue.enable: false
2019-03-15T15:47:53.056575+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - dead_letter_queue.max_bytes: 1073741824
2019-03-15T15:47:53.056653+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - slowlog.threshold.warn: -1
2019-03-15T15:47:53.057137+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.056 [LogStash::Runner] runner - slowlog.threshold.info: -1
2019-03-15T15:47:53.057218+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - slowlog.threshold.debug: -1
2019-03-15T15:47:53.057290+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - slowlog.threshold.trace: -1
2019-03-15T15:47:53.057541+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - keystore.classname: "org.logstash.secret.store.backend.JavaKeyStore"
2019-03-15T15:47:53.057620+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - *keystore.file: "./logstash.keystore" (default: "/usr/share/logstash/config/logstash.keystore")
2019-03-15T15:47:53.057717+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - path.queue: "/usr/share/logstash/data/queue"
2019-03-15T15:47:53.057795+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - path.dead_letter_queue: "/usr/share/logstash/data/dead_letter_queue"
2019-03-15T15:47:53.057882+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - *path.settings: "." (default: "/usr/share/logstash/config")
2019-03-15T15:47:53.057947+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - path.logs: "/usr/share/logstash/logs"
2019-03-15T15:47:53.058033+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.057 [LogStash::Runner] runner - xpack.management.enabled: false
2019-03-15T15:47:53.058115+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.logstash.poll_interval: 5000000000
2019-03-15T15:47:53.058196+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.pipeline.id: ["main"]
2019-03-15T15:47:53.058581+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.elasticsearch.username: "logstash_system"
2019-03-15T15:47:53.058677+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.elasticsearch.url: ["https://localhost:9200"]
2019-03-15T15:47:53.058752+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.elasticsearch.ssl.verification_mode: "certificate"
2019-03-15T15:47:53.058852+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.management.elasticsearch.sniffing: false
2019-03-15T15:47:53.058928+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.monitoring.enabled: false
2019-03-15T15:47:53.059008+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.058 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.url: ["http://localhost:9200"]
2019-03-15T15:47:53.059093+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.collection.interval: 10000000000
2019-03-15T15:47:53.059175+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.collection.timeout_interval: 600000000000
2019-03-15T15:47:53.059246+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.username: "logstash_system"
2019-03-15T15:47:53.059338+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.ssl.verification_mode: "certificate"
2019-03-15T15:47:53.059411+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.sniffing: false
2019-03-15T15:47:53.059508+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.collection.pipeline.details.enabled: true
2019-03-15T15:47:53.059574+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - xpack.monitoring.collection.config.enabled: true
2019-03-15T15:47:53.059661+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - node.uuid: ""
2019-03-15T15:47:53.059735+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.059 [LogStash::Runner] runner - --------------- Logstash Settings -------------------
2019-03-15T15:47:53.107195+00:00 app[web.1]: [WARN ] 2019-03-15 15:47:53.106 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
2019-03-15T15:47:53.122868+00:00 app[web.1]: [INFO ] 2019-03-15 15:47:53.122 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.6.1"}
2019-03-15T15:47:53.157840+00:00 app[web.1]: [INFO ] 2019-03-15 15:47:53.157 [LogStash::Runner] agent - No persistent UUID file found. Generating new UUID {:uuid=>"e09b5267-413c-4598-90bb-ca912960a62a", :path=>"/usr/share/logstash/data/uuid"}
2019-03-15T15:47:53.165900+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.165 [LogStash::Runner] agent - Setting global FieldReference parsing mode: COMPAT
2019-03-15T15:47:53.188036+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.187 [LogStash::Runner] agent - Setting up metric collection
2019-03-15T15:47:53.343508+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.342 [LogStash::Runner] os - Starting {:polling_interval=>5, :polling_timeout=>120}
2019-03-15T15:47:53.657869+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.657 [LogStash::Runner] jvm - Starting {:polling_interval=>5, :polling_timeout=>120}
2019-03-15T15:47:53.802087+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.801 [LogStash::Runner] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:47:53.809059+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.808 [LogStash::Runner] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:47:53.830469+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.830 [LogStash::Runner] persistentqueue - Starting {:polling_interval=>5, :polling_timeout=>120}
2019-03-15T15:47:53.844845+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.844 [LogStash::Runner] deadletterqueue - Starting {:polling_interval=>5, :polling_timeout=>120}
2019-03-15T15:47:53.913850+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.913 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Starting agent
2019-03-15T15:47:53.994298+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.992 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] configpathloader - Skipping the following files while reading config since they don't match the specified glob pattern {:files=>["/usr/share/logstash/CONTRIBUTORS", "/usr/share/logstash/Gemfile", "/usr/share/logstash/Gemfile.lock", "/usr/share/logstash/LICENSE.txt", "/usr/share/logstash/NOTICE.TXT", "/usr/share/logstash/bin", "/usr/share/logstash/config", "/usr/share/logstash/data", "/usr/share/logstash/lib", "/usr/share/logstash/logstash-core", "/usr/share/logstash/logstash-core-plugin-api", "/usr/share/logstash/logstash.yml", "/usr/share/logstash/modules", "/usr/share/logstash/pipeline", "/usr/share/logstash/template.json", "/usr/share/logstash/tools", "/usr/share/logstash/vendor", "/usr/share/logstash/x-pack"]}
2019-03-15T15:47:53.997468+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:53.997 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] configpathloader - Reading config file {:config_file=>"/usr/share/logstash/logstash.conf"}
2019-03-15T15:47:54.061183+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:54.059 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Converging pipelines state {:actions_count=>1}
2019-03-15T15:47:54.078377+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:54.073 [Converge PipelineAction::Create<main>] agent - Executing action {:action=>LogStash::PipelineAction::Create/pipeline_id:main}
2019-03-15T15:47:58.939850+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:58.939 [pool-3-thread-2] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:47:58.941673+00:00 app[web.1]: [DEBUG] 2019-03-15 15:47:58.941 [pool-3-thread-2] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:01.255911+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:01.255 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"tcp", :type=>"input", :class=>LogStash::Inputs::Tcp}
2019-03-15T15:48:01.324014+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:01.322 [Converge PipelineAction::Create<main>] tcp - Replacing `${PORT}` with actual value
2019-03-15T15:48:01.329582+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:01.329 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:02.143780+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.143 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"line", :type=>"codec", :class=>LogStash::Codecs::Line}
2019-03-15T15:48:02.173822+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.173 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@id = "line_a169b908-c5b8-461e-bc74-4677b03ede73"
2019-03-15T15:48:02.174634+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.174 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@enable_metric = true
2019-03-15T15:48:02.174754+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.174 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@charset = "UTF-8"
2019-03-15T15:48:02.174876+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.174 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@delimiter = "\n"
2019-03-15T15:48:02.201800+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.201 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@port = 48857
2019-03-15T15:48:02.202092+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.201 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@id = "1a66f1736c8fbba0d27e30383af519fb6ae5e6ea95c0fa687da2299d8ddeffee"
2019-03-15T15:48:02.202214+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.202 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@enable_metric = true
2019-03-15T15:48:02.212256+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.212 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@codec = <LogStash::Codecs::Line id=>"line_a169b908-c5b8-461e-bc74-4677b03ede73", enable_metric=>true, charset=>"UTF-8", delimiter=>"\n">
2019-03-15T15:48:02.212846+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.212 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@add_field = {}
2019-03-15T15:48:02.212936+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.212 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@host = "0.0.0.0"
2019-03-15T15:48:02.213055+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.212 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@mode = "server"
2019-03-15T15:48:02.213254+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.213 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@proxy_protocol = false
2019-03-15T15:48:02.213318+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.213 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@ssl_enable = false
2019-03-15T15:48:02.213390+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.213 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@ssl_verify = true
2019-03-15T15:48:02.215848+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.215 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@ssl_key_passphrase = <password>
2019-03-15T15:48:02.215951+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.215 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@ssl_extra_chain_certs = []
2019-03-15T15:48:02.216041+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.215 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@ssl_certificate_authorities = []
2019-03-15T15:48:02.216115+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.216 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@tcp_keep_alive = false
2019-03-15T15:48:02.216192+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.216 [Converge PipelineAction::Create<main>] tcp - config LogStash::Inputs::Tcp/@dns_reverse_lookup_enabled = true
2019-03-15T15:48:02.260213+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.259 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"grok", :type=>"filter", :class=>LogStash::Filters::Grok}
2019-03-15T15:48:02.275743+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.275 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@match = {"message"=>"%{SYSLOG5424PRI:pri}%{NUMBER:rfc_version} %{TIMESTAMP_ISO8601:timestamp} %{USERNAME:app} %{USERNAME:dyno} %{USERNAME:proc} - Event %{USERNAME:event_id}: %{GREEDYDATA:message}"}
2019-03-15T15:48:02.276146+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@id = "9f3159f023af8fb70dcd3ec1412cb2275777bc496f81800c6490c6c550c20595"
2019-03-15T15:48:02.276263+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@overwrite = ["message"]
2019-03-15T15:48:02.276364+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@remove_field = ["pri", "syslog5424_pri", "rfc_version", "dyno", "proc"]
2019-03-15T15:48:02.276471+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@enable_metric = true
2019-03-15T15:48:02.276610+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@add_tag = []
2019-03-15T15:48:02.276701+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@remove_tag = []
2019-03-15T15:48:02.276864+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@add_field = {}
2019-03-15T15:48:02.276965+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@periodic_flush = false
2019-03-15T15:48:02.277058+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.276 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@patterns_dir = []
2019-03-15T15:48:02.277129+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@pattern_definitions = {}
2019-03-15T15:48:02.277228+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@patterns_files_glob = "*"
2019-03-15T15:48:02.277298+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@break_on_match = true
2019-03-15T15:48:02.277395+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@named_captures_only = true
2019-03-15T15:48:02.277472+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@keep_empty_captures = false
2019-03-15T15:48:02.277586+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@tag_on_failure = ["_grokparsefailure"]
2019-03-15T15:48:02.277656+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@timeout_millis = 30000
2019-03-15T15:48:02.277764+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.277 [Converge PipelineAction::Create<main>] grok - config LogStash::Filters::Grok/@tag_on_timeout = "_groktimeout"
2019-03-15T15:48:02.288582+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.288 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"drop", :type=>"filter", :class=>LogStash::Filters::Drop}
2019-03-15T15:48:02.295797+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.295 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@id = "7f14389c539a488c8046f13e3c26b5848b9acc817b27b7fe5a4e9a378d68d5d0"
2019-03-15T15:48:02.296003+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.295 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@enable_metric = true
2019-03-15T15:48:02.296080+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@add_tag = []
2019-03-15T15:48:02.296141+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@remove_tag = []
2019-03-15T15:48:02.296210+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@add_field = {}
2019-03-15T15:48:02.296274+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@remove_field = []
2019-03-15T15:48:02.296346+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@periodic_flush = false
2019-03-15T15:48:02.296420+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.296 [Converge PipelineAction::Create<main>] drop - config LogStash::Filters::Drop/@percentage = 100
2019-03-15T15:48:02.300767+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.300 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"clone", :type=>"filter", :class=>LogStash::Filters::Clone}
2019-03-15T15:48:02.306937+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.306 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@clones = ["for_es", "for_s3"]
2019-03-15T15:48:02.307130+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@id = "9a06fdb63115f8b1f1add75221e5097a06f15fa28df29a23be1b44fc1390feff"
2019-03-15T15:48:02.307195+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@enable_metric = true
2019-03-15T15:48:02.307264+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@add_tag = []
2019-03-15T15:48:02.307321+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@remove_tag = []
2019-03-15T15:48:02.307400+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@add_field = {}
2019-03-15T15:48:02.307468+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@remove_field = []
2019-03-15T15:48:02.307535+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.307 [Converge PipelineAction::Create<main>] clone - config LogStash::Filters::Clone/@periodic_flush = false
2019-03-15T15:48:02.313692+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.313 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"json", :type=>"filter", :class=>LogStash::Filters::Json}
2019-03-15T15:48:02.320327+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@remove_field = ["message", "@version", "port", "host", "type", "@timestamp"]
2019-03-15T15:48:02.320519+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@source = "message"
2019-03-15T15:48:02.320577+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@id = "f64fe830634adde3aae761873356aaafe98e85c3d0bfb746859d0b25948f3650"
2019-03-15T15:48:02.320657+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@enable_metric = true
2019-03-15T15:48:02.320722+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@add_tag = []
2019-03-15T15:48:02.320784+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@remove_tag = []
2019-03-15T15:48:02.320855+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@add_field = {}
2019-03-15T15:48:02.320919+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@periodic_flush = false
2019-03-15T15:48:02.321003+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.320 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@tag_on_failure = ["_jsonparsefailure"]
2019-03-15T15:48:02.321070+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.321 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@skip_on_invalid_json = false
2019-03-15T15:48:02.327489+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.327 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"mutate", :type=>"filter", :class=>LogStash::Filters::Mutate}
2019-03-15T15:48:02.335104+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.334 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@add_field = {"[@metadata][type]"=>"for_es"}
2019-03-15T15:48:02.335298+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@id = "4359a4b5a23d156a65a9147a5c31b816e9b72b7c7bfe0465e729239f3c585979"
2019-03-15T15:48:02.335372+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@enable_metric = true
2019-03-15T15:48:02.335439+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@add_tag = []
2019-03-15T15:48:02.335504+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@remove_tag = []
2019-03-15T15:48:02.335599+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@remove_field = []
2019-03-15T15:48:02.335690+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.335 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@periodic_flush = false
2019-03-15T15:48:02.342944+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.342 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@source = "message"
2019-03-15T15:48:02.343126+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@id = "f845da13982dc3a4b9a25735ed8938f427f1a2a056deac6e163339ddbd54a0f8"
2019-03-15T15:48:02.343223+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@enable_metric = true
2019-03-15T15:48:02.343315+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@add_tag = []
2019-03-15T15:48:02.343372+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@remove_tag = []
2019-03-15T15:48:02.343445+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@add_field = {}
2019-03-15T15:48:02.343505+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@remove_field = []
2019-03-15T15:48:02.343576+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@periodic_flush = false
2019-03-15T15:48:02.343674+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@tag_on_failure = ["_jsonparsefailure"]
2019-03-15T15:48:02.343734+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.343 [Converge PipelineAction::Create<main>] json - config LogStash::Filters::Json/@skip_on_invalid_json = false
2019-03-15T15:48:02.347974+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.347 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@add_field = {"[@metadata][type]"=>"for_s3"}
2019-03-15T15:48:02.348049+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.347 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@id = "4eff132d93981b42a8a2bc655c8bfdfcc572c6d76bb30b6071e067cf7f5c2710"
2019-03-15T15:48:02.348122+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.348 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@enable_metric = true
2019-03-15T15:48:02.348187+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.348 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@add_tag = []
2019-03-15T15:48:02.348248+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.348 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@remove_tag = []
2019-03-15T15:48:02.348313+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.348 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@remove_field = []
2019-03-15T15:48:02.348377+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.348 [Converge PipelineAction::Create<main>] mutate - config LogStash::Filters::Mutate/@periodic_flush = false
2019-03-15T15:48:02.352022+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.351 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"rubydebug", :type=>"codec", :class=>LogStash::Codecs::RubyDebug}
2019-03-15T15:48:02.356655+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.356 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@metadata = true
2019-03-15T15:48:02.356733+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.356 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@id = "48dccc1e-66a5-4eb8-ba39-b02abdc383b2"
2019-03-15T15:48:02.356808+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:02.356 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@enable_metric = true
2019-03-15T15:48:04.320421+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.319 [pool-3-thread-1] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:04.321819+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.321 [pool-3-thread-1] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:04.342670+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.342 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"stdout", :type=>"output", :class=>LogStash::Outputs::Stdout}
2019-03-15T15:48:04.383723+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.383 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@metadata = true
2019-03-15T15:48:04.383778+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.383 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@id = "48dccc1e-66a5-4eb8-ba39-b02abdc383b2"
2019-03-15T15:48:04.383875+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.383 [Converge PipelineAction::Create<main>] rubydebug - config LogStash::Codecs::RubyDebug/@enable_metric = true
2019-03-15T15:48:04.394406+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.394 [Converge PipelineAction::Create<main>] stdout - config LogStash::Outputs::Stdout/@codec = <LogStash::Codecs::RubyDebug metadata=>true, id=>"48dccc1e-66a5-4eb8-ba39-b02abdc383b2", enable_metric=>true>
2019-03-15T15:48:04.394474+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.394 [Converge PipelineAction::Create<main>] stdout - config LogStash::Outputs::Stdout/@id = "f5262c0ed903575391099b14911bbbbec5e5b6012a0dbfad08d14d64d6bfc920"
2019-03-15T15:48:04.394567+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.394 [Converge PipelineAction::Create<main>] stdout - config LogStash::Outputs::Stdout/@enable_metric = true
2019-03-15T15:48:04.394647+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.394 [Converge PipelineAction::Create<main>] stdout - config LogStash::Outputs::Stdout/@workers = 1
2019-03-15T15:48:04.401422+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.401 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"elasticsearch", :type=>"output", :class=>LogStash::Outputs::ElasticSearch}
2019-03-15T15:48:04.408135+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.407 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_PWD}` with actual value
2019-03-15T15:48:04.408810+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:04.408 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:05.020372+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:05.020 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_HOST}` with actual value
2019-03-15T15:48:05.024093+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:05.023 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:05.590455+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:05.590 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_PORT}` with actual value
2019-03-15T15:48:05.590958+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:05.590 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:06.115075+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.114 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_USER}` with actual value
2019-03-15T15:48:06.115481+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.115 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:06.667451+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.667 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"plain", :type=>"codec", :class=>LogStash::Codecs::Plain}
2019-03-15T15:48:06.674909+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.674 [Converge PipelineAction::Create<main>] plain - config LogStash::Codecs::Plain/@id = "plain_16a8fcb7-b691-4b1b-87d3-8702eb913caf"
2019-03-15T15:48:06.675000+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.674 [Converge PipelineAction::Create<main>] plain - config LogStash::Codecs::Plain/@enable_metric = true
2019-03-15T15:48:06.675117+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.675 [Converge PipelineAction::Create<main>] plain - config LogStash::Codecs::Plain/@charset = "UTF-8"
2019-03-15T15:48:06.685000+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.684 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_PWD}` with actual value
2019-03-15T15:48:06.685184+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:06.685 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:07.247810+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:07.247 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_HOST}` with actual value
2019-03-15T15:48:07.248120+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:07.248 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:07.764819+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:07.764 [Converge PipelineAction::Create<main>] elasticsearch - Replacing `${ES_PORT}` with actual value
2019-03-15T15:48:07.764980+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:07.764 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:08.324449+00:00 app[web.1]: [WARN ] 2019-03-15 15:48:08.307 [Converge PipelineAction::Create<main>] elasticsearch - You are using a deprecated config setting "document_type" set in elasticsearch. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature If you have any questions about this, please visit the #logstash channel on freenode irc. {:name=>"document_type", :plugin=><LogStash::Outputs::ElasticSearch template=>"./template.json", password=><password>, template_name=>"events", hosts=>[//ivy-613842576.us-east-1.bonsaisearch.net:9200], index=>"events", id=>"baa12fca65270d27e5d2c5961e7c1c4a6afba976fc1d28b15d4d524171875b82", document_id=>"%{[event_id]}", user=>"61gr4rzdme", failure_type_logging_whitelist=>["document_already_exists_exception"], document_type=>"logs", enable_metric=>true, codec=><LogStash::Codecs::Plain id=>"plain_16a8fcb7-b691-4b1b-87d3-8702eb913caf", enable_metric=>true, charset=>"UTF-8">, workers=>1, manage_template=>true, template_overwrite=>false, doc_as_upsert=>false, script_type=>"inline", script_lang=>"painless", script_var_name=>"event", scripted_upsert=>false, retry_initial_interval=>2, retry_max_interval=>64, retry_on_conflict=>1, ilm_enabled=>false, ilm_rollover_alias=>"logstash", ilm_pattern=>"{now/d}-000001", ilm_policy=>"logstash-policy", action=>"index", ssl_certificate_verification=>true, sniffing=>false, sniffing_delay=>5, timeout=>60, pool_max=>1000, pool_max_per_route=>100, resurrect_delay=>5, validate_after_inactivity=>10000, http_compression=>false>}
2019-03-15T15:48:08.325130+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@template = "./template.json"
2019-03-15T15:48:08.325264+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@password = <password>
2019-03-15T15:48:08.325328+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@template_name = "events"
2019-03-15T15:48:08.325690+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@hosts = [//ivy-613842576.us-east-1.bonsaisearch.net:9200]
2019-03-15T15:48:08.325767+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@index = "events"
2019-03-15T15:48:08.325835+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@id = "baa12fca65270d27e5d2c5961e7c1c4a6afba976fc1d28b15d4d524171875b82"
2019-03-15T15:48:08.325915+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@document_id = "%{[event_id]}"
2019-03-15T15:48:08.325966+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@user = "61gr4rzdme"
2019-03-15T15:48:08.326042+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.325 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@failure_type_logging_whitelist = ["document_already_exists_exception"]
2019-03-15T15:48:08.326110+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@document_type = "logs"
2019-03-15T15:48:08.326177+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@enable_metric = true
2019-03-15T15:48:08.326336+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@codec = <LogStash::Codecs::Plain id=>"plain_16a8fcb7-b691-4b1b-87d3-8702eb913caf", enable_metric=>true, charset=>"UTF-8">
2019-03-15T15:48:08.326424+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@workers = 1
2019-03-15T15:48:08.326478+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@manage_template = true
2019-03-15T15:48:08.326543+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@template_overwrite = false
2019-03-15T15:48:08.326610+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@parent = nil
2019-03-15T15:48:08.326670+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@join_field = nil
2019-03-15T15:48:08.326754+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@upsert = ""
2019-03-15T15:48:08.326829+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@doc_as_upsert = false
2019-03-15T15:48:08.326912+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@script = ""
2019-03-15T15:48:08.326959+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@script_type = "inline"
2019-03-15T15:48:08.327036+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.326 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@script_lang = "painless"
2019-03-15T15:48:08.327104+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@script_var_name = "event"
2019-03-15T15:48:08.327181+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@scripted_upsert = false
2019-03-15T15:48:08.327253+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@retry_initial_interval = 2
2019-03-15T15:48:08.327322+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@retry_max_interval = 64
2019-03-15T15:48:08.327407+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@retry_on_conflict = 1
2019-03-15T15:48:08.327644+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@pipeline = nil
2019-03-15T15:48:08.327821+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@ilm_enabled = false
2019-03-15T15:48:08.327927+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@ilm_rollover_alias = "logstash"
2019-03-15T15:48:08.327998+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.327 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@ilm_pattern = "{now/d}-000001"
2019-03-15T15:48:08.328067+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@ilm_policy = "logstash-policy"
2019-03-15T15:48:08.328126+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@action = "index"
2019-03-15T15:48:08.328202+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@ssl_certificate_verification = true
2019-03-15T15:48:08.328285+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@sniffing = false
2019-03-15T15:48:08.328353+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@sniffing_delay = 5
2019-03-15T15:48:08.328421+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@timeout = 60
2019-03-15T15:48:08.328487+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@pool_max = 1000
2019-03-15T15:48:08.328563+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@pool_max_per_route = 100
2019-03-15T15:48:08.328622+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@resurrect_delay = 5
2019-03-15T15:48:08.328690+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@validate_after_inactivity = 10000
2019-03-15T15:48:08.328758+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@http_compression = false
2019-03-15T15:48:08.328833+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:08.328 [Converge PipelineAction::Create<main>] elasticsearch - config LogStash::Outputs::ElasticSearch/@custom_headers = {}
2019-03-15T15:48:09.330302+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:09.330 [pool-3-thread-1] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:09.330738+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:09.330 [pool-3-thread-1] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:14.339552+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:14.339 [pool-3-thread-1] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:14.340146+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:14.340 [pool-3-thread-1] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:18.751711+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:18.751 [Converge PipelineAction::Create<main>] registry - On demand adding plugin to the registry {:name=>"s3", :type=>"output", :class=>LogStash::Outputs::S3}
2019-03-15T15:48:18.772958+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:18.772 [Converge PipelineAction::Create<main>] s3 - Replacing `${AWS_ACCESS_KEY_ID}` with actual value
2019-03-15T15:48:18.773843+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:18.773 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:19.940397+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:19.940 [Converge PipelineAction::Create<main>] s3 - Replacing `${AWS_SECRET_ACCESS_KEY}` with actual value
2019-03-15T15:48:19.940812+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:19.940 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:21.050747+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:21.050 [Converge PipelineAction::Create<main>] s3 - Replacing `${AWS_DEFAULT_REGION}` with actual value
2019-03-15T15:48:21.050971+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:21.050 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:22.182435+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.174 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:22.184604+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.184 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:22.235747+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.235 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@id = "line_00640811-aef4-46ca-a054-06c0046fd3c5"
2019-03-15T15:48:22.235944+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.235 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@enable_metric = true
2019-03-15T15:48:22.236070+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.236 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@charset = "UTF-8"
2019-03-15T15:48:22.236169+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.236 [Converge PipelineAction::Create<main>] line - config LogStash::Codecs::Line/@delimiter = "\n"
2019-03-15T15:48:22.242167+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.242 [Converge PipelineAction::Create<main>] s3 - Replacing `${AWS_SECRET_ACCESS_KEY}` with actual value
2019-03-15T15:48:22.242406+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:22.242 [Converge PipelineAction::Create<main>] SecretStoreFactory - Attempting to exists or secret store with implementation: org.logstash.secret.store.backend.JavaKeyStore
2019-03-15T15:48:23.348442+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@access_key_id = "AKIAJMEGVPRB35UFDAFA"
2019-03-15T15:48:23.348542+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@bucket = "vpal-app-event-logs"
2019-03-15T15:48:23.348650+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@prefix = "%{[app]}/%{[deploy_env]}/%{+YYYY}/%{+MM}/%{+dd}"
2019-03-15T15:48:23.348740+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@rotation_strategy = "size_and_time"
2019-03-15T15:48:23.348845+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@time_file = 15
2019-03-15T15:48:23.349048+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.348 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@secret_access_key = <password>
2019-03-15T15:48:23.349140+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.349 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@id = "f2319d0dc53010370d534372d106e63c28877634997a1a7728664b3b99268e66"
2019-03-15T15:48:23.349242+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.349 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@region = "us-east-1"
2019-03-15T15:48:23.349330+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.349 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@enable_metric = true
2019-03-15T15:48:23.349445+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.349 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@tags = []
2019-03-15T15:48:23.360497+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@codec = <LogStash::Codecs::Line id=>"line_00640811-aef4-46ca-a054-06c0046fd3c5", enable_metric=>true, charset=>"UTF-8", delimiter=>"\n">
2019-03-15T15:48:23.360617+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@workers = 1
2019-03-15T15:48:23.360734+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@role_session_name = "logstash"
2019-03-15T15:48:23.360847+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@additional_settings = {}
2019-03-15T15:48:23.360932+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@size_file = 5242880
2019-03-15T15:48:23.361044+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.360 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@restore = true
2019-03-15T15:48:23.361129+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@canned_acl = "private"
2019-03-15T15:48:23.361236+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@server_side_encryption = false
2019-03-15T15:48:23.361339+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@server_side_encryption_algorithm = "AES256"
2019-03-15T15:48:23.361425+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@storage_class = "STANDARD"
2019-03-15T15:48:23.361537+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@temporary_directory = "/tmp/logstash"
2019-03-15T15:48:23.361656+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@upload_workers_count = 1
2019-03-15T15:48:23.361742+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@upload_queue_size = 2
2019-03-15T15:48:23.361851+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@encoding = "none"
2019-03-15T15:48:23.361943+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.361 [Converge PipelineAction::Create<main>] s3 - config LogStash::Outputs::S3/@validate_credentials_on_root_bucket = true
2019-03-15T15:48:23.428676+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:23.428 [Converge PipelineAction::Create<main>] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
2019-03-15T15:48:23.517008+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:23.516 [[main]-pipeline-manager] elasticsearch - Normalizing http path {:path=>nil, :normalized=>nil}
2019-03-15T15:48:23.992308+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:23.981 [[main]-pipeline-manager] elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://61gr4rzdme:xxxxxx@ivy-613842576.us-east-1.bonsaisearch.net:9200/]}}
2019-03-15T15:48:24.004795+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:24.001 [[main]-pipeline-manager] elasticsearch - Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://61gr4rzdme:xxxxxx@ivy-613842576.us-east-1.bonsaisearch.net:9200/, :path=>"/"}
2019-03-15T15:48:24.365366+00:00 app[web.1]: [WARN ] 2019-03-15 15:48:24.365 [[main]-pipeline-manager] elasticsearch - Restored connection to ES instance {:url=>"http://61gr4rzdme:xxxxxx@ivy-613842576.us-east-1.bonsaisearch.net:9200/"}
2019-03-15T15:48:24.593712+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:24.593 [[main]-pipeline-manager] elasticsearch - ES Output version determined {:es_version=>6}
2019-03-15T15:48:24.600289+00:00 app[web.1]: [WARN ] 2019-03-15 15:48:24.597 [[main]-pipeline-manager] elasticsearch - Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
2019-03-15T15:48:24.642932+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:24.642 [[main]-pipeline-manager] elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//ivy-613842576.us-east-1.bonsaisearch.net:9200"]}
2019-03-15T15:48:24.660380+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:24.659 [Ruby-0-Thread-5: :1] elasticsearch - Using mapping template from {:path=>"./template.json"}
2019-03-15T15:48:24.683548+00:00 app[web.1]: [INFO ] 2019-03-15 15:48:24.683 [Ruby-0-Thread-5: :1] elasticsearch - Attempting to install template {:manage_template=>{"template"=>"events", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"logs"=>{"properties"=>{"timestamp"=>{"type"=>"date"}, "app"=>{"type"=>"keyword"}, "deploy_env"=>{"type"=>"keyword"}, "kind"=>{"type"=>"keyword"}}}}}}
2019-03-15T15:48:24.772855+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:24.772 [Ruby-0-Thread-5: :1] elasticsearch - Found existing Elasticsearch template. Skipping template management {:name=>"events"}
2019-03-15T15:48:27.193781+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:27.193 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:27.194118+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:27.194 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:32.207602+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:32.200 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:32.208452+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:32.208 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:37.215739+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:37.215 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:37.216049+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:37.215 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:42.221684+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:42.221 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:42.221971+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:42.221 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:47.227409+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:47.227 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:47.227638+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:47.227 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:52.232338+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:52.232 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:52.232867+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:52.232 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:48:57.239865+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:57.239 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:48:57.240146+00:00 app[web.1]: [DEBUG] 2019-03-15 15:48:57.240 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:49:02.245910+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:02.245 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:49:02.245915+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:02.245 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:49:07.253487+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:07.253 [pool-3-thread-3] jvm - collector name {:name=>"ParNew"}
2019-03-15T15:49:07.254893+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:07.253 [pool-3-thread-3] jvm - collector name {:name=>"ConcurrentMarkSweep"}
2019-03-15T15:49:08.079974+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.079 [[main]-pipeline-manager] s3 - Start periodic rotation check
2019-03-15T15:49:08.121631+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.121 [[main]-pipeline-manager] grok - Grok patterns path {:paths=>["/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns", "/usr/share/logstash/patterns/*"]}
2019-03-15T15:49:08.133193+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.132 [[main]-pipeline-manager] grok - Grok patterns path {:paths=>[]}
2019-03-15T15:49:08.136893+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.136 [[main]-pipeline-manager] grok - Match data {:match=>{"message"=>"%{SYSLOG5424PRI:pri}%{NUMBER:rfc_version} %{TIMESTAMP_ISO8601:timestamp} %{USERNAME:app} %{USERNAME:dyno} %{USERNAME:proc} - Event %{USERNAME:event_id}: %{GREEDYDATA:message}"}}
2019-03-15T15:49:08.149441+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.149 [[main]-pipeline-manager] grok - regexp: /message {:pattern=>"%{SYSLOG5424PRI:pri}%{NUMBER:rfc_version} %{TIMESTAMP_ISO8601:timestamp} %{USERNAME:app} %{USERNAME:dyno} %{USERNAME:proc} - Event %{USERNAME:event_id}: %{GREEDYDATA:message}"}
2019-03-15T15:49:08.216129+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.215 [[main]-pipeline-manager] grok - Adding pattern {"MCOLLECTIVEAUDIT"=>"%{TIMESTAMP_ISO8601:timestamp}:"}
2019-03-15T15:49:08.218219+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.218 [[main]-pipeline-manager] grok - Adding pattern {"REDISTIMESTAMP"=>"%{MONTHDAY} %{MONTH} %{TIME}"}
2019-03-15T15:49:08.218364+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.218 [[main]-pipeline-manager] grok - Adding pattern {"REDISLOG"=>"\\[%{POSINT:pid}\\] %{REDISTIMESTAMP:timestamp} \\* "}
2019-03-15T15:49:08.218502+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.218 [[main]-pipeline-manager] grok - Adding pattern {"REDISMONLOG"=>"%{NUMBER:timestamp} \\[%{INT:database} %{IP:client}:%{NUMBER:port}\\] \"%{WORD:command}\"\\s?%{GREEDYDATA:params}"}
2019-03-15T15:49:08.218912+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.218 [[main]-pipeline-manager] grok - Adding pattern {"MONGO_LOG"=>"%{SYSLOGTIMESTAMP:timestamp} \\[%{WORD:component}\\] %{GREEDYDATA:message}"}
2019-03-15T15:49:08.219052+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.218 [[main]-pipeline-manager] grok - Adding pattern {"MONGO_QUERY"=>"\\{ (?<={ ).*(?= } ntoreturn:) \\}"}
2019-03-15T15:49:08.219219+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.219 [[main]-pipeline-manager] grok - Adding pattern {"MONGO_SLOWQUERY"=>"%{WORD} %{MONGO_WORDDASH:database}\\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?<duration>[0-9]+)ms"}
2019-03-15T15:49:08.219329+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.219 [[main]-pipeline-manager] grok - Adding pattern {"MONGO_WORDDASH"=>"\\b[\\w-]+\\b"}
2019-03-15T15:49:08.219459+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.219 [[main]-pipeline-manager] grok - Adding pattern {"MONGO3_SEVERITY"=>"\\w"}
2019-03-15T15:49:08.219609+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.219 [[main]-pipeline-manager] grok - Adding pattern {"MONGO3_COMPONENT"=>"%{WORD}|-"}
2019-03-15T15:49:08.219752+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.219 [[main]-pipeline-manager] grok - Adding pattern {"MONGO3_LOG"=>"%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\\[%{DATA:context}\\])? %{GREEDYDATA:message}"}
2019-03-15T15:49:08.220544+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.220 [[main]-pipeline-manager] grok - Adding pattern {"POSTGRESQL"=>"%{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid}"}
2019-03-15T15:49:08.221517+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.221 [[main]-pipeline-manager] grok - Adding pattern {"RUUID"=>"\\h{32}"}
2019-03-15T15:49:08.221664+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.221 [[main]-pipeline-manager] grok - Adding pattern {"RCONTROLLER"=>"(?<controller>[^#]+)#(?<action>\\w+)"}
2019-03-15T15:49:08.221843+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.221 [[main]-pipeline-manager] grok - Adding pattern {"RAILS3HEAD"=>"(?m)Started %{WORD:verb} \"%{URIPATHPARAM:request}\" for %{IPORHOST:clientip} at (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE})"}
2019-03-15T15:49:08.221974+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.221 [[main]-pipeline-manager] grok - Adding pattern {"RPROCESSING"=>"\\W*Processing by %{RCONTROLLER} as (?<format>\\S+)(?:\\W*Parameters: {%{DATA:params}}\\W*)?"}
2019-03-15T15:49:08.222121+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"RAILS3FOOT"=>"Completed %{NUMBER:response}%{DATA} in %{NUMBER:totalms}ms %{RAILS3PROFILE}%{GREEDYDATA}"}
2019-03-15T15:49:08.222247+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"RAILS3PROFILE"=>"(?:\\(Views: %{NUMBER:viewms}ms \\| ActiveRecord: %{NUMBER:activerecordms}ms|\\(ActiveRecord: %{NUMBER:activerecordms}ms)?"}
2019-03-15T15:49:08.222413+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"RAILS3"=>"%{RAILS3HEAD}(?:%{RPROCESSING})?(?<context>(?:%{DATA}\\n)*)(?:%{RAILS3FOOT})?"}
2019-03-15T15:49:08.222674+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOG5424PRINTASCII"=>"[!-~]+"}
2019-03-15T15:49:08.222889+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGBASE2"=>"(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource}+(?: %{SYSLOGPROG}:|)"}
2019-03-15T15:49:08.223060+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.222 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGPAMSESSION"=>"%{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\\(%{DATA:pam_caller}\\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?"}
2019-03-15T15:49:08.223192+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"CRON_ACTION"=>"[A-Z ]+"}
2019-03-15T15:49:08.223324+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"CRONLOG"=>"%{SYSLOGBASE} \\(%{USER:user}\\) %{CRON_ACTION:action} \\(%{DATA:message}\\)"}
2019-03-15T15:49:08.223465+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGLINE"=>"%{SYSLOGBASE2} %{GREEDYDATA:message}"}
2019-03-15T15:49:08.223647+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOG5424PRI"=>"<%{NONNEGINT:syslog5424_pri}>"}
2019-03-15T15:49:08.223770+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOG5424SD"=>"\\[%{DATA}\\]+"}
2019-03-15T15:49:08.223927+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.223 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOG5424BASE"=>"%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)"}
2019-03-15T15:49:08.224090+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOG5424LINE"=>"%{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}"}
2019-03-15T15:49:08.224377+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"USERNAME"=>"[a-zA-Z0-9._-]+"}
2019-03-15T15:49:08.224504+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"USER"=>"%{USERNAME}"}
2019-03-15T15:49:08.224623+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"EMAILLOCALPART"=>"[a-zA-Z][a-zA-Z0-9_.+-=:]+"}
2019-03-15T15:49:08.224761+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"EMAILADDRESS"=>"%{EMAILLOCALPART}@%{HOSTNAME}"}
2019-03-15T15:49:08.224901+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.224 [[main]-pipeline-manager] grok - Adding pattern {"INT"=>"(?:[+-]?(?:[0-9]+))"}
2019-03-15T15:49:08.225131+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"BASE10NUM"=>"(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"}
2019-03-15T15:49:08.225267+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"NUMBER"=>"(?:%{BASE10NUM})"}
2019-03-15T15:49:08.225416+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"BASE16NUM"=>"(?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))"}
2019-03-15T15:49:08.225546+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"BASE16FLOAT"=>"\\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\\.[0-9A-Fa-f]*)?)|(?:\\.[0-9A-Fa-f]+)))\\b"}
2019-03-15T15:49:08.225708+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"POSINT"=>"\\b(?:[1-9][0-9]*)\\b"}
2019-03-15T15:49:08.225848+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"NONNEGINT"=>"\\b(?:[0-9]+)\\b"}
2019-03-15T15:49:08.225973+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.225 [[main]-pipeline-manager] grok - Adding pattern {"WORD"=>"\\b\\w+\\b"}
2019-03-15T15:49:08.226095+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"NOTSPACE"=>"\\S+"}
2019-03-15T15:49:08.226210+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"SPACE"=>"\\s*"}
2019-03-15T15:49:08.226328+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"DATA"=>".*?"}
2019-03-15T15:49:08.226451+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"GREEDYDATA"=>".*"}
2019-03-15T15:49:08.226586+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"QUOTEDSTRING"=>"(?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))"}
2019-03-15T15:49:08.226732+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"UUID"=>"[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}"}
2019-03-15T15:49:08.226904+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"URN"=>"urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+"}
2019-03-15T15:49:08.227076+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.226 [[main]-pipeline-manager] grok - Adding pattern {"MAC"=>"(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})"}
2019-03-15T15:49:08.227199+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.227 [[main]-pipeline-manager] grok - Adding pattern {"CISCOMAC"=>"(?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})"}
2019-03-15T15:49:08.227317+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.227 [[main]-pipeline-manager] grok - Adding pattern {"WINDOWSMAC"=>"(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})"}
2019-03-15T15:49:08.227429+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.227 [[main]-pipeline-manager] grok - Adding pattern {"COMMONMAC"=>"(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})"}
2019-03-15T15:49:08.227759+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.227 [[main]-pipeline-manager] grok - Adding pattern {"IPV6"=>"((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?"}
2019-03-15T15:49:08.227989+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.227 [[main]-pipeline-manager] grok - Adding pattern {"IPV4"=>"(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])"}
2019-03-15T15:49:08.228110+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"IP"=>"(?:%{IPV6}|%{IPV4})"}
2019-03-15T15:49:08.228251+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"HOSTNAME"=>"\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)"}
2019-03-15T15:49:08.228369+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"IPORHOST"=>"(?:%{IP}|%{HOSTNAME})"}
2019-03-15T15:49:08.228486+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"HOSTPORT"=>"%{IPORHOST}:%{POSINT}"}
2019-03-15T15:49:08.228628+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"PATH"=>"(?:%{UNIXPATH}|%{WINPATH})"}
2019-03-15T15:49:08.228762+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"UNIXPATH"=>"(/([\\w_%!$@:.,+~-]+|\\\\.)*)+"}
2019-03-15T15:49:08.228873+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"TTY"=>"(?:/dev/(pts|tty([pq])?)(\\w+)?/?(?:[0-9]+))"}
2019-03-15T15:49:08.228994+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.228 [[main]-pipeline-manager] grok - Adding pattern {"WINPATH"=>"(?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+"}
2019-03-15T15:49:08.229110+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URIPROTO"=>"[A-Za-z]([A-Za-z0-9+\\-.]+)+"}
2019-03-15T15:49:08.229235+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URIHOST"=>"%{IPORHOST}(?::%{POSINT:port})?"}
2019-03-15T15:49:08.229365+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URIPATH"=>"(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-]*)+"}
2019-03-15T15:49:08.229495+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URIPARAM"=>"\\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\\-\\[\\]<>]*"}
2019-03-15T15:49:08.229632+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URIPATHPARAM"=>"%{URIPATH}(?:%{URIPARAM})?"}
2019-03-15T15:49:08.229770+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"URI"=>"%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?"}
2019-03-15T15:49:08.230114+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.229 [[main]-pipeline-manager] grok - Adding pattern {"MONTH"=>"\\b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\\b"}
2019-03-15T15:49:08.230258+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"MONTHNUM"=>"(?:0?[1-9]|1[0-2])"}
2019-03-15T15:49:08.230409+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"MONTHNUM2"=>"(?:0[1-9]|1[0-2])"}
2019-03-15T15:49:08.230540+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"MONTHDAY"=>"(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])"}
2019-03-15T15:49:08.230740+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"DAY"=>"(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)"}
2019-03-15T15:49:08.230882+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"YEAR"=>"(?>\\d\\d){1,2}"}
2019-03-15T15:49:08.231010+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.230 [[main]-pipeline-manager] grok - Adding pattern {"HOUR"=>"(?:2[0123]|[01]?[0-9])"}
2019-03-15T15:49:08.231130+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"MINUTE"=>"(?:[0-5][0-9])"}
2019-03-15T15:49:08.231264+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"SECOND"=>"(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)"}
2019-03-15T15:49:08.231387+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"TIME"=>"(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])"}
2019-03-15T15:49:08.231520+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"DATE_US"=>"%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}"}
2019-03-15T15:49:08.231650+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"DATE_EU"=>"%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}"}
2019-03-15T15:49:08.231778+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"ISO8601_TIMEZONE"=>"(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))"}
2019-03-15T15:49:08.231905+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"ISO8601_SECOND"=>"(?:%{SECOND}|60)"}
2019-03-15T15:49:08.232056+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.231 [[main]-pipeline-manager] grok - Adding pattern {"TIMESTAMP_ISO8601"=>"%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?"}
2019-03-15T15:49:08.232194+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATE"=>"%{DATE_US}|%{DATE_EU}"}
2019-03-15T15:49:08.232317+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATESTAMP"=>"%{DATE}[- ]%{TIME}"}
2019-03-15T15:49:08.232450+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"TZ"=>"(?:[APMCE][SD]T|UTC)"}
2019-03-15T15:49:08.232570+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATESTAMP_RFC822"=>"%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}"}
2019-03-15T15:49:08.232702+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATESTAMP_RFC2822"=>"%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}"}
2019-03-15T15:49:08.232839+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATESTAMP_OTHER"=>"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}"}
2019-03-15T15:49:08.232964+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.232 [[main]-pipeline-manager] grok - Adding pattern {"DATESTAMP_EVENTLOG"=>"%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}"}
2019-03-15T15:49:08.233119+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.233 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGTIMESTAMP"=>"%{MONTH} +%{MONTHDAY} %{TIME}"}
2019-03-15T15:49:08.233237+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.233 [[main]-pipeline-manager] grok - Adding pattern {"PROG"=>"[\\x21-\\x5a\\x5c\\x5e-\\x7e]+"}
2019-03-15T15:49:08.233370+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.233 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGPROG"=>"%{PROG:program}(?:\\[%{POSINT:pid}\\])?"}
2019-03-15T15:49:08.233498+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.233 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGHOST"=>"%{IPORHOST}"}
2019-03-15T15:49:08.233994+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.233 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGFACILITY"=>"<%{NONNEGINT:facility}.%{NONNEGINT:priority}>"}
2019-03-15T15:49:08.234115+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.234 [[main]-pipeline-manager] grok - Adding pattern {"HTTPDATE"=>"%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}"}
2019-03-15T15:49:08.234263+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.234 [[main]-pipeline-manager] grok - Adding pattern {"QS"=>"%{QUOTEDSTRING}"}
2019-03-15T15:49:08.234419+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.234 [[main]-pipeline-manager] grok - Adding pattern {"SYSLOGBASE"=>"%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:"}
2019-03-15T15:49:08.234608+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.234 [[main]-pipeline-manager] grok - Adding pattern {"LOGLEVEL"=>"([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)"}
2019-03-15T15:49:08.234987+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.234 [[main]-pipeline-manager] grok - Adding pattern {"RT_FLOW_EVENT"=>"(RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY)"}
2019-03-15T15:49:08.235217+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.235 [[main]-pipeline-manager] grok - Adding pattern {"RT_FLOW1"=>"%{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \\d+\\(%{DATA:sent}\\) \\d+\\(%{DATA:received}\\) %{INT:elapsed-time} .*"}
2019-03-15T15:49:08.235426+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.235 [[main]-pipeline-manager] grok - Adding pattern {"RT_FLOW2"=>"%{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .*"}
2019-03-15T15:49:08.235603+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.235 [[main]-pipeline-manager] grok - Adding pattern {"RT_FLOW3"=>"%{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\\(\\d\\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .*"}
2019-03-15T15:49:08.235892+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.235 [[main]-pipeline-manager] grok - Adding pattern {"RUBY_LOGLEVEL"=>"(?:DEBUG|FATAL|ERROR|WARN|INFO)"}
2019-03-15T15:49:08.236047+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.235 [[main]-pipeline-manager] grok - Adding pattern {"RUBY_LOGGER"=>"[DFEWI], \\[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message}"}
2019-03-15T15:49:08.236299+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.236 [[main]-pipeline-manager] grok - Adding pattern {"MAVEN_VERSION"=>"(?:(\\d+)\\.)?(?:(\\d+)\\.)?(\\*|\\d+)(?:[.-](RELEASE|SNAPSHOT))?"}
2019-03-15T15:49:08.236653+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.236 [[main]-pipeline-manager] grok - Adding pattern {"NETSCREENSESSIONLOG"=>"%{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason}"}
2019-03-15T15:49:08.236814+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.236 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_TAGGED_SYSLOG"=>"^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?: %%{CISCOTAG:ciscotag}:"}
2019-03-15T15:49:08.236949+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.236 [[main]-pipeline-manager] grok - Adding pattern {"CISCOTIMESTAMP"=>"%{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME}"}
2019-03-15T15:49:08.237079+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCOTAG"=>"[A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+)"}
2019-03-15T15:49:08.237225+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_ACTION"=>"Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted"}
2019-03-15T15:49:08.237376+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_REASON"=>"Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\\s*)*"}
2019-03-15T15:49:08.237521+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_DIRECTION"=>"Inbound|inbound|Outbound|outbound"}
2019-03-15T15:49:08.237630+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_INTERVAL"=>"first hit|%{INT}-second interval"}
2019-03-15T15:49:08.237756+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCO_XLATE_TYPE"=>"static|dynamic"}
2019-03-15T15:49:08.237906+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW104001"=>"\\((?:Primary|Secondary)\\) Switching to ACTIVE - %{GREEDYDATA:switch_reason}"}
2019-03-15T15:49:08.238043+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.237 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW104002"=>"\\((?:Primary|Secondary)\\) Switching to STANDBY - %{GREEDYDATA:switch_reason}"}
2019-03-15T15:49:08.238183+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW104003"=>"\\((?:Primary|Secondary)\\) Switching to FAILED\\."}
2019-03-15T15:49:08.238312+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW104004"=>"\\((?:Primary|Secondary)\\) Switching to OK\\."}
2019-03-15T15:49:08.238458+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW105003"=>"\\((?:Primary|Secondary)\\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} waiting"}
2019-03-15T15:49:08.238597+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW105004"=>"\\((?:Primary|Secondary)\\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} normal"}
2019-03-15T15:49:08.238764+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW105005"=>"\\((?:Primary|Secondary)\\) Lost Failover communications with mate on [Ii]nterface %{GREEDYDATA:interface_name}"}
2019-03-15T15:49:08.238920+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW105008"=>"\\((?:Primary|Secondary)\\) Testing [Ii]nterface %{GREEDYDATA:interface_name}"}
2019-03-15T15:49:08.239072+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.238 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW105009"=>"\\((?:Primary|Secondary)\\) Testing on [Ii]nterface %{GREEDYDATA:interface_name} (?:Passed|Failed)"}
2019-03-15T15:49:08.239313+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.239 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106001"=>"%{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface}"}
2019-03-15T15:49:08.239478+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.239 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106006_106007_106010"=>"%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\\(%{DATA:src_fwuser}\\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\\(%{DATA:dst_fwuser}\\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason})"}
2019-03-15T15:49:08.239761+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.239 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106014"=>"%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\\(%{DATA:src_fwuser}\\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\\(%{DATA:dst_fwuser}\\))? \\(type %{INT:icmp_type}, code %{INT:icmp_code}\\)"}
2019-03-15T15:49:08.239928+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.239 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106015"=>"%{CISCO_ACTION:action} %{WORD:protocol} \\(%{DATA:policy_id}\\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface}"}
2019-03-15T15:49:08.240080+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.239 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106021"=>"%{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}"}
2019-03-15T15:49:08.240366+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.240 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106023"=>"%{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src_interface}:%{DATA:src_ip}(/%{INT:src_port})?(\\(%{DATA:src_fwuser}\\))? dst %{DATA:dst_interface}:%{DATA:dst_ip}(/%{INT:dst_port})?(\\(%{DATA:dst_fwuser}\\))?( \\(type %{INT:icmp_type}, code %{INT:icmp_code}\\))? by access-group \"?%{DATA:policy_id}\"? \\[%{DATA:hashcode1}, %{DATA:hashcode2}\\]"}
2019-03-15T15:49:08.240552+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.240 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106100_2_3"=>"access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\\(%{INT:src_port}\\) -> %{DATA:dst_interface}/%{IP:dst_ip}\\(%{INT:dst_port}\\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \\[%{DATA:hashcode1}, %{DATA:hashcode2}\\]"}
2019-03-15T15:49:08.240752+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.240 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW106100"=>"access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\\(%{INT:src_port}\\)(\\(%{DATA:src_fwuser}\\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\\(%{INT:dst_port}\\)(\\(%{DATA:src_fwuser}\\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \\[%{DATA:hashcode1}, %{DATA:hashcode2}\\]"}
2019-03-15T15:49:08.240884+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.240 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW304001"=>"%{IP:src_ip}(\\(%{DATA:src_fwuser}\\))? Accessed URL %{IP:dst_ip}:%{GREEDYDATA:dst_url}"}
2019-03-15T15:49:08.241031+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.240 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW110002"=>"%{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}"}
2019-03-15T15:49:08.241160+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW302010"=>"%{INT:connection_count} in use, %{INT:connection_count_max} most used"}
2019-03-15T15:49:08.241366+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW302013_302014_302015_302016"=>"%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \\(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\\))?(\\(%{DATA:src_fwuser}\\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \\(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\\))?(\\(%{DATA:dst_fwuser}\\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \\(%{DATA:user}\\))?"}
2019-03-15T15:49:08.241525+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW302020_302021"=>"%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\\(%{DATA:fwuser}\\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \\(%{DATA:user}\\))?"}
2019-03-15T15:49:08.241697+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW305011"=>"%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\\(%{DATA:src_fwuser}\\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port}"}
2019-03-15T15:49:08.241841+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW313001_313004_313008"=>"%{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})?"}
2019-03-15T15:49:08.242047+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.241 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW313005"=>"%{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\\(%{DATA:err_src_fwuser}\\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\\(%{DATA:err_dst_fwuser}\\))? \\(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\\) on %{DATA:interface} interface\\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\\(%{DATA:orig_src_fwuser}\\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\\(%{DATA:orig_dst_fwuser}\\))?"}
2019-03-15T15:49:08.242182+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW321001"=>"Resource '%{WORD:resource_name}' limit of %{POSINT:resource_limit} reached for system"}
2019-03-15T15:49:08.242333+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW402117"=>"%{WORD:protocol}: Received a non-IPSec packet \\(protocol= %{WORD:orig_protocol}\\) from %{IP:src_ip} to %{IP:dst_ip}"}
2019-03-15T15:49:08.242479+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW402119"=>"%{WORD:protocol}: Received an %{WORD:orig_protocol} packet \\(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\\) from %{IP:src_ip} \\(user= %{DATA:user}\\) to %{IP:dst_ip} that failed anti-replay checking"}
2019-03-15T15:49:08.242647+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW419001"=>"%{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason}"}
2019-03-15T15:49:08.242825+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW419002"=>"%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number"}
2019-03-15T15:49:08.242961+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.242 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW500004"=>"%{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}"}
2019-03-15T15:49:08.243167+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.243 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW602303_602304"=>"%{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \\(SPI= %{DATA:spi}\\) between %{IP:src_ip} and %{IP:dst_ip} \\(user= %{DATA:user}\\) has been %{CISCO_ACTION:action}"}
2019-03-15T15:49:08.243315+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.243 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW710001_710002_710003_710005_710006"=>"%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}"}
2019-03-15T15:49:08.243480+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.243 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW713172"=>"Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\\s+Remote end\\s*%{DATA:is_remote_natted}\\s*behind a NAT device\\s+This\\s+end\\s*%{DATA:is_local_natted}\\s*behind a NAT device"}
2019-03-15T15:49:08.243701+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.243 [[main]-pipeline-manager] grok - Adding pattern {"CISCOFW733100"=>"\\[\\s*%{DATA:drop_type}\\s*\\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count}"}
2019-03-15T15:49:08.243936+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.243 [[main]-pipeline-manager] grok - Adding pattern {"SHOREWALL"=>"(%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)"}
2019-03-15T15:49:08.244161+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.244 [[main]-pipeline-manager] grok - Adding pattern {"SFW2"=>"((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\\s*%{HOSTNAME}\\s*kernel\\S+\\s*%{NAGIOSTIME}\\s*SFW2\\-INext\\-%{NOTSPACE:nf_action}\\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\\s*%{USERNAME:nf_out_interface})|(\\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\\s*)).*SRC=%{IP:nf_src_ip}\\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())"}
2019-03-15T15:49:08.244460+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.244 [[main]-pipeline-manager] grok - Adding pattern {"MCOLLECTIVE"=>"., \\[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\\]%{SPACE}%{LOGLEVEL:event_level}"}
2019-03-15T15:49:08.244591+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.244 [[main]-pipeline-manager] grok - Adding pattern {"MCOLLECTIVEAUDIT"=>"%{TIMESTAMP_ISO8601:timestamp}:"}
2019-03-15T15:49:08.244832+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.244 [[main]-pipeline-manager] grok - Adding pattern {"BIND9_TIMESTAMP"=>"%{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}"}
2019-03-15T15:49:08.244998+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.244 [[main]-pipeline-manager] grok - Adding pattern {"BIND9"=>"%{BIND9_TIMESTAMP:timestamp} queries: %{LOGLEVEL:loglevel}: client %{IP:clientip}#%{POSINT:clientport} \\(%{GREEDYDATA:query}\\): query: %{GREEDYDATA:query} IN %{GREEDYDATA:querytype} \\(%{IP:dns}\\)"}
2019-03-15T15:49:08.245265+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.245 [[main]-pipeline-manager] grok - Adding pattern {"HTTPDUSER"=>"%{EMAILADDRESS}|%{USER}"}
2019-03-15T15:49:08.245404+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.245 [[main]-pipeline-manager] grok - Adding pattern {"HTTPDERROR_DATE"=>"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"}
2019-03-15T15:49:08.245999+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.245 [[main]-pipeline-manager] grok - Adding pattern {"HTTPD_COMMONLOG"=>"%{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)"}
2019-03-15T15:49:08.246133+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"HTTPD_COMBINEDLOG"=>"%{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}"}
2019-03-15T15:49:08.246290+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"HTTPD20_ERRORLOG"=>"\\[%{HTTPDERROR_DATE:timestamp}\\] \\[%{LOGLEVEL:loglevel}\\] (?:\\[client %{IPORHOST:clientip}\\] ){0,1}%{GREEDYDATA:message}"}
2019-03-15T15:49:08.246422+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"HTTPD24_ERRORLOG"=>"\\[%{HTTPDERROR_DATE:timestamp}\\] \\[%{WORD:module}:%{LOGLEVEL:loglevel}\\] \\[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\\]( \\(%{POSINT:proxy_errorcode}\\)%{DATA:proxy_message}:)?( \\[client %{IPORHOST:clientip}:%{POSINT:clientport}\\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}"}
2019-03-15T15:49:08.246553+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"HTTPD_ERRORLOG"=>"%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}"}
2019-03-15T15:49:08.246680+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"COMMONAPACHELOG"=>"%{HTTPD_COMMONLOG}"}
2019-03-15T15:49:08.246830+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"COMBINEDAPACHELOG"=>"%{HTTPD_COMBINEDLOG}"}
2019-03-15T15:49:08.247088+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.246 [[main]-pipeline-manager] grok - Adding pattern {"JAVACLASS"=>"(?:[a-zA-Z$_][a-zA-Z$_0-9]*\\.)*[a-zA-Z$_][a-zA-Z$_0-9]*"}
2019-03-15T15:49:08.247216+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVAFILE"=>"(?:[A-Za-z0-9_. -]+)"}
2019-03-15T15:49:08.247334+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVAMETHOD"=>"(?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)"}
2019-03-15T15:49:08.247457+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVASTACKTRACEPART"=>"%{SPACE}at %{JAVACLASS:class}\\.%{JAVAMETHOD:method}\\(%{JAVAFILE:file}(?::%{NUMBER:line})?\\)"}
2019-03-15T15:49:08.247580+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVATHREAD"=>"(?:[A-Z]{2}-Processor[\\d]+)"}
2019-03-15T15:49:08.247688+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVACLASS"=>"(?:[a-zA-Z0-9-]+\\.)+[A-Za-z0-9$]+"}
2019-03-15T15:49:08.247797+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVAFILE"=>"(?:[A-Za-z0-9_.-]+)"}
2019-03-15T15:49:08.247905+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"JAVALOGMESSAGE"=>"(.*)"}
2019-03-15T15:49:08.248028+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.247 [[main]-pipeline-manager] grok - Adding pattern {"CATALINA_DATESTAMP"=>"%{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)"}
2019-03-15T15:49:08.248161+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"TOMCAT_DATESTAMP"=>"20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}"}
2019-03-15T15:49:08.248273+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"CATALINALOG"=>"%{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}"}
2019-03-15T15:49:08.248398+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"TOMCATLOG"=>"%{TOMCAT_DATESTAMP:timestamp} \\| %{LOGLEVEL:level} \\| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}"}
2019-03-15T15:49:08.248635+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_MSGID"=>"[0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}"}
2019-03-15T15:49:08.248746+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_FLAGS"=>"(<=|[-=>*]>|[*]{2}|==)"}
2019-03-15T15:49:08.248862+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_DATE"=>"%{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time}"}
2019-03-15T15:49:08.248969+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_PID"=>"\\[%{POSINT}\\]"}
2019-03-15T15:49:08.249091+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.248 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_QT"=>"((\\d+y)?(\\d+w)?(\\d+d)?(\\d+h)?(\\d+m)?(\\d+s)?)"}
2019-03-15T15:49:08.249349+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_EXCLUDE_TERMS"=>"(Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)"}
2019-03-15T15:49:08.249474+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_REMOTE_HOST"=>"(H=(%{NOTSPACE:remote_hostname} )?(\\(%{NOTSPACE:remote_heloname}\\) )?\\[%{IP:remote_host}\\])"}
2019-03-15T15:49:08.249591+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_INTERFACE"=>"(I=\\[%{IP:exim_interface}\\](:%{NUMBER:exim_interface_port}))"}
2019-03-15T15:49:08.249700+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_PROTOCOL"=>"(P=%{NOTSPACE:protocol})"}
2019-03-15T15:49:08.249806+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_MSG_SIZE"=>"(S=%{NUMBER:exim_msg_size})"}
2019-03-15T15:49:08.249917+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_HEADER_ID"=>"(id=%{NOTSPACE:exim_header_id})"}
2019-03-15T15:49:08.250024+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.249 [[main]-pipeline-manager] grok - Adding pattern {"EXIM_SUBJECT"=>"(T=%{QS:exim_subject})"}
2019-03-15T15:49:08.250338+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.250 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYTIME"=>"(?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9])"}
2019-03-15T15:49:08.250452+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.250 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYDATE"=>"%{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}"}
2019-03-15T15:49:08.250597+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.250 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYCAPTUREDREQUESTHEADERS"=>"%{DATA:captured_request_headers}"}
2019-03-15T15:49:08.250774+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.250 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYCAPTUREDRESPONSEHEADERS"=>"%{DATA:captured_response_headers}"}
2019-03-15T15:49:08.251082+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.250 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYHTTPBASE"=>"%{IP:client_ip}:%{INT:client_port} \\[%{HAPROXYDATE:accept_date}\\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\\{%{HAPROXYCAPTUREDREQUESTHEADERS}\\})?( )?(\\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\\})?( )?\"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?\""}
2019-03-15T15:49:08.251213+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.251 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYHTTP"=>"(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}"}
2019-03-15T15:49:08.251401+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.251 [[main]-pipeline-manager] grok - Adding pattern {"HAPROXYTCP"=>"(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \\[%{HAPROXYDATE:accept_date}\\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue}"}
2019-03-15T15:49:08.251710+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.251 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOSTIME"=>"\\[%{NUMBER:nagios_epoch}\\]"}
2019-03-15T15:49:08.251834+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.251 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_CURRENT_SERVICE_STATE"=>"CURRENT SERVICE STATE"}
2019-03-15T15:49:08.251950+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.251 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_CURRENT_HOST_STATE"=>"CURRENT HOST STATE"}
2019-03-15T15:49:08.252073+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_SERVICE_NOTIFICATION"=>"SERVICE NOTIFICATION"}
2019-03-15T15:49:08.252183+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_HOST_NOTIFICATION"=>"HOST NOTIFICATION"}
2019-03-15T15:49:08.252302+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_SERVICE_ALERT"=>"SERVICE ALERT"}
2019-03-15T15:49:08.252410+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_HOST_ALERT"=>"HOST ALERT"}
2019-03-15T15:49:08.252535+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_SERVICE_FLAPPING_ALERT"=>"SERVICE FLAPPING ALERT"}
2019-03-15T15:49:08.252644+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_HOST_FLAPPING_ALERT"=>"HOST FLAPPING ALERT"}
2019-03-15T15:49:08.252766+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT"=>"SERVICE DOWNTIME ALERT"}
2019-03-15T15:49:08.252878+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_HOST_DOWNTIME_ALERT"=>"HOST DOWNTIME ALERT"}
2019-03-15T15:49:08.253001+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.252 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_PASSIVE_SERVICE_CHECK"=>"PASSIVE SERVICE CHECK"}
2019-03-15T15:49:08.253111+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_PASSIVE_HOST_CHECK"=>"PASSIVE HOST CHECK"}
2019-03-15T15:49:08.253244+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_SERVICE_EVENT_HANDLER"=>"SERVICE EVENT HANDLER"}
2019-03-15T15:49:08.253363+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_HOST_EVENT_HANDLER"=>"HOST EVENT HANDLER"}
2019-03-15T15:49:08.253480+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_EXTERNAL_COMMAND"=>"EXTERNAL COMMAND"}
2019-03-15T15:49:08.253595+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TYPE_TIMEPERIOD_TRANSITION"=>"TIMEPERIOD TRANSITION"}
2019-03-15T15:49:08.253742+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_DISABLE_SVC_CHECK"=>"DISABLE_SVC_CHECK"}
2019-03-15T15:49:08.253856+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_ENABLE_SVC_CHECK"=>"ENABLE_SVC_CHECK"}
2019-03-15T15:49:08.253964+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_DISABLE_HOST_CHECK"=>"DISABLE_HOST_CHECK"}
2019-03-15T15:49:08.254072+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.253 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_ENABLE_HOST_CHECK"=>"ENABLE_HOST_CHECK"}
2019-03-15T15:49:08.254182+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT"=>"PROCESS_SERVICE_CHECK_RESULT"}
2019-03-15T15:49:08.254296+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_PROCESS_HOST_CHECK_RESULT"=>"PROCESS_HOST_CHECK_RESULT"}
2019-03-15T15:49:08.254404+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME"=>"SCHEDULE_SERVICE_DOWNTIME"}
2019-03-15T15:49:08.254515+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_SCHEDULE_HOST_DOWNTIME"=>"SCHEDULE_HOST_DOWNTIME"}
2019-03-15T15:49:08.254625+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS"=>"DISABLE_HOST_SVC_NOTIFICATIONS"}
2019-03-15T15:49:08.254757+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS"=>"ENABLE_HOST_SVC_NOTIFICATIONS"}
2019-03-15T15:49:08.254894+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS"=>"DISABLE_HOST_NOTIFICATIONS"}
2019-03-15T15:49:08.255004+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.254 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS"=>"ENABLE_HOST_NOTIFICATIONS"}
2019-03-15T15:49:08.255118+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS"=>"DISABLE_SVC_NOTIFICATIONS"}
2019-03-15T15:49:08.255234+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS"=>"ENABLE_SVC_NOTIFICATIONS"}
2019-03-15T15:49:08.255363+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_WARNING"=>"Warning:%{SPACE}%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.255505+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_CURRENT_SERVICE_STATE"=>"%{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.255648+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_CURRENT_HOST_STATE"=>"%{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.255780+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_SERVICE_NOTIFICATION"=>"%{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.255917+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_HOST_NOTIFICATION"=>"%{NAGIOS_TYPE_HOST_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.256056+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.255 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_SERVICE_ALERT"=>"%{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.256201+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_HOST_ALERT"=>"%{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.256361+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_SERVICE_FLAPPING_ALERT"=>"%{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.256516+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_HOST_FLAPPING_ALERT"=>"%{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}"}
2019-03-15T15:49:08.256669+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_SERVICE_DOWNTIME_ALERT"=>"%{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}"}
2019-03-15T15:49:08.256790+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_HOST_DOWNTIME_ALERT"=>"%{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}"}
2019-03-15T15:49:08.256922+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_PASSIVE_SERVICE_CHECK"=>"%{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}"}
2019-03-15T15:49:08.257044+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.256 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_PASSIVE_HOST_CHECK"=>"%{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}"}
2019-03-15T15:49:08.257252+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_SERVICE_EVENT_HANDLER"=>"%{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}"}
2019-03-15T15:49:08.257390+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_HOST_EVENT_HANDLER"=>"%{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}"}
2019-03-15T15:49:08.257541+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_TIMEPERIOD_TRANSITION"=>"%{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}"}
2019-03-15T15:49:08.257694+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_DISABLE_SVC_CHECK"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}"}
2019-03-15T15:49:08.257834+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_DISABLE_HOST_CHECK"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}"}
2019-03-15T15:49:08.257974+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.257 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_ENABLE_SVC_CHECK"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}"}
2019-03-15T15:49:08.258110+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_ENABLE_HOST_CHECK"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}"}
2019-03-15T15:49:08.258257+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}"}
2019-03-15T15:49:08.258404+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}"}
2019-03-15T15:49:08.258540+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}"}
2019-03-15T15:49:08.258680+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}"}
2019-03-15T15:49:08.258840+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service}"}
2019-03-15T15:49:08.258979+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.258 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}"}
2019-03-15T15:49:08.259124+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.259 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}"}
2019-03-15T15:49:08.259240+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.259 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service}"}
2019-03-15T15:49:08.259439+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.259 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME"=>"%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment}"}
2019-03-15T15:49:08.259735+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.259 [[main]-pipeline-manager] grok - Adding pattern {"NAGIOSLOGLINE"=>"%{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS})"}
2019-03-15T15:49:08.260007+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.259 [[main]-pipeline-manager] grok - Adding pattern {"S3_REQUEST_LINE"=>"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})"}
2019-03-15T15:49:08.260237+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"S3_ACCESS_LOG"=>"%{WORD:owner} %{NOTSPACE:bucket} \\[%{HTTPDATE:timestamp}\\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:\"%{S3_REQUEST_LINE}\"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:\"?%{QS:agent}\"?|-) (?:-|%{NOTSPACE:version_id})"}
2019-03-15T15:49:08.260359+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"ELB_URIPATHPARAM"=>"%{URIPATH:path}(?:%{URIPARAM:params})?"}
2019-03-15T15:49:08.260493+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"ELB_URI"=>"%{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{ELB_URIPATHPARAM})?"}
2019-03-15T15:49:08.260645+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"ELB_REQUEST_LINE"=>"(?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})"}
2019-03-15T15:49:08.260841+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"ELB_ACCESS_LOG"=>"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} \"%{ELB_REQUEST_LINE}\""}
2019-03-15T15:49:08.261054+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.260 [[main]-pipeline-manager] grok - Adding pattern {"CLOUDFRONT_ACCESS_LOG"=>"(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\\t%{TIME})\\t%{WORD:x_edge_location}\\t(?:%{NUMBER:sc_bytes:int}|-)\\t%{IPORHOST:clientip}\\t%{WORD:cs_method}\\t%{HOSTNAME:cs_host}\\t%{NOTSPACE:cs_uri_stem}\\t%{NUMBER:sc_status:int}\\t%{GREEDYDATA:referrer}\\t%{GREEDYDATA:agent}\\t%{GREEDYDATA:cs_uri_query}\\t%{GREEDYDATA:cookies}\\t%{WORD:x_edge_result_type}\\t%{NOTSPACE:x_edge_request_id}\\t%{HOSTNAME:x_host_header}\\t%{URIPROTO:cs_protocol}\\t%{INT:cs_bytes:int}\\t%{GREEDYDATA:time_taken:float}\\t%{GREEDYDATA:x_forwarded_for}\\t%{GREEDYDATA:ssl_protocol}\\t%{GREEDYDATA:ssl_cipher}\\t%{GREEDYDATA:x_edge_response_result_type}"}
2019-03-15T15:49:08.261387+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.261 [[main]-pipeline-manager] grok - Adding pattern {"SQUID3"=>"%{NUMBER:timestamp}\\s+%{NUMBER:duration}\\s%{IP:client_address}\\s%{WORD:cache_result}/%{POSINT:status_code}\\s%{NUMBER:bytes}\\s%{WORD:request_method}\\s%{NOTSPACE:url}\\s(%{NOTSPACE:user}|-)\\s%{WORD:hierarchy_code}/%{IPORHOST:server}\\s%{NOTSPACE:content_type}"}
2019-03-15T15:49:08.261733+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.261 [[main]-pipeline-manager] grok - Adding pattern {"BRO_HTTP"=>"%{NUMBER:ts}\\t%{NOTSPACE:uid}\\t%{IP:orig_h}\\t%{INT:orig_p}\\t%{IP:resp_h}\\t%{INT:resp_p}\\t%{INT:trans_depth}\\t%{GREEDYDATA:method}\\t%{GREEDYDATA:domain}\\t%{GREEDYDATA:uri}\\t%{GREEDYDATA:referrer}\\t%{GREEDYDATA:user_agent}\\t%{NUMBER:request_body_len}\\t%{NUMBER:response_body_len}\\t%{GREEDYDATA:status_code}\\t%{GREEDYDATA:status_msg}\\t%{GREEDYDATA:info_code}\\t%{GREEDYDATA:info_msg}\\t%{GREEDYDATA:filename}\\t%{GREEDYDATA:bro_tags}\\t%{GREEDYDATA:username}\\t%{GREEDYDATA:password}\\t%{GREEDYDATA:proxied}\\t%{GREEDYDATA:orig_fuids}\\t%{GREEDYDATA:orig_mime_types}\\t%{GREEDYDATA:resp_fuids}\\t%{GREEDYDATA:resp_mime_types}"}
2019-03-15T15:49:08.261983+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.261 [[main]-pipeline-manager] grok - Adding pattern {"BRO_DNS"=>"%{NUMBER:ts}\\t%{NOTSPACE:uid}\\t%{IP:orig_h}\\t%{INT:orig_p}\\t%{IP:resp_h}\\t%{INT:resp_p}\\t%{WORD:proto}\\t%{INT:trans_id}\\t%{GREEDYDATA:query}\\t%{GREEDYDATA:qclass}\\t%{GREEDYDATA:qclass_name}\\t%{GREEDYDATA:qtype}\\t%{GREEDYDATA:qtype_name}\\t%{GREEDYDATA:rcode}\\t%{GREEDYDATA:rcode_name}\\t%{GREEDYDATA:AA}\\t%{GREEDYDATA:TC}\\t%{GREEDYDATA:RD}\\t%{GREEDYDATA:RA}\\t%{GREEDYDATA:Z}\\t%{GREEDYDATA:answers}\\t%{GREEDYDATA:TTLs}\\t%{GREEDYDATA:rejected}"}
2019-03-15T15:49:08.262166+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BRO_CONN"=>"%{NUMBER:ts}\\t%{NOTSPACE:uid}\\t%{IP:orig_h}\\t%{INT:orig_p}\\t%{IP:resp_h}\\t%{INT:resp_p}\\t%{WORD:proto}\\t%{GREEDYDATA:service}\\t%{NUMBER:duration}\\t%{NUMBER:orig_bytes}\\t%{NUMBER:resp_bytes}\\t%{GREEDYDATA:conn_state}\\t%{GREEDYDATA:local_orig}\\t%{GREEDYDATA:missed_bytes}\\t%{GREEDYDATA:history}\\t%{GREEDYDATA:orig_pkts}\\t%{GREEDYDATA:orig_ip_bytes}\\t%{GREEDYDATA:resp_pkts}\\t%{GREEDYDATA:resp_ip_bytes}\\t%{GREEDYDATA:tunnel_parents}"}
2019-03-15T15:49:08.262407+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BRO_FILES"=>"%{NUMBER:ts}\\t%{NOTSPACE:fuid}\\t%{IP:tx_hosts}\\t%{IP:rx_hosts}\\t%{NOTSPACE:conn_uids}\\t%{GREEDYDATA:source}\\t%{GREEDYDATA:depth}\\t%{GREEDYDATA:analyzers}\\t%{GREEDYDATA:mime_type}\\t%{GREEDYDATA:filename}\\t%{GREEDYDATA:duration}\\t%{GREEDYDATA:local_orig}\\t%{GREEDYDATA:is_orig}\\t%{GREEDYDATA:seen_bytes}\\t%{GREEDYDATA:total_bytes}\\t%{GREEDYDATA:missing_bytes}\\t%{GREEDYDATA:overflow_bytes}\\t%{GREEDYDATA:timedout}\\t%{GREEDYDATA:parent_fuid}\\t%{GREEDYDATA:md5}\\t%{GREEDYDATA:sha1}\\t%{GREEDYDATA:sha256}\\t%{GREEDYDATA:extracted}"}
2019-03-15T15:49:08.262668+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_TIMESTAMP"=>"%{MONTHDAY}-%{MONTH} %{HOUR}:%{MINUTE}"}
2019-03-15T15:49:08.262820+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_HOST"=>"[a-zA-Z0-9-]+"}
2019-03-15T15:49:08.262927+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_VOLUME"=>"%{USER}"}
2019-03-15T15:49:08.263054+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.262 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_DEVICE"=>"%{USER}"}
2019-03-15T15:49:08.263156+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_DEVICEPATH"=>"%{UNIXPATH}"}
2019-03-15T15:49:08.263281+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_CAPACITY"=>"%{INT}{1,3}(,%{INT}{3})*"}
2019-03-15T15:49:08.263387+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_VERSION"=>"%{USER}"}
2019-03-15T15:49:08.263487+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_JOB"=>"%{USER}"}
2019-03-15T15:49:08.263624+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_MAX_CAPACITY"=>"User defined maximum volume capacity %{BACULA_CAPACITY} exceeded on device \\\"%{BACULA_DEVICE:device}\\\" \\(%{BACULA_DEVICEPATH}\\)"}
2019-03-15T15:49:08.263749+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_END_VOLUME"=>"End of medium on Volume \\\"%{BACULA_VOLUME:volume}\\\" Bytes=%{BACULA_CAPACITY} Blocks=%{BACULA_CAPACITY} at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}."}
2019-03-15T15:49:08.263861+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NEW_VOLUME"=>"Created new Volume \\\"%{BACULA_VOLUME:volume}\\\" in catalog."}
2019-03-15T15:49:08.263981+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.263 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NEW_LABEL"=>"Labeled new Volume \\\"%{BACULA_VOLUME:volume}\\\" on device \\\"%{BACULA_DEVICE:device}\\\" \\(%{BACULA_DEVICEPATH}\\)."}
2019-03-15T15:49:08.264104+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_WROTE_LABEL"=>"Wrote label to prelabeled Volume \\\"%{BACULA_VOLUME:volume}\\\" on device \\\"%{BACULA_DEVICE}\\\" \\(%{BACULA_DEVICEPATH}\\)"}
2019-03-15T15:49:08.264243+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NEW_MOUNT"=>"New volume \\\"%{BACULA_VOLUME:volume}\\\" mounted on device \\\"%{BACULA_DEVICE:device}\\\" \\(%{BACULA_DEVICEPATH}\\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}."}
2019-03-15T15:49:08.264358+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOOPEN"=>"\\s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}"}
2019-03-15T15:49:08.264469+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOOPENDIR"=>"\\s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}"}
2019-03-15T15:49:08.264588+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOSTAT"=>"\\s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}"}
2019-03-15T15:49:08.264727+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOJOBS"=>"There are no more Jobs associated with Volume \\\"%{BACULA_VOLUME:volume}\\\". Marking it purged."}
2019-03-15T15:49:08.264846+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_ALL_RECORDS_PRUNED"=>"All records pruned from Volume \\\"%{BACULA_VOLUME:volume}\\\"; marking it \\\"Purged\\\""}
2019-03-15T15:49:08.264960+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.264 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_BEGIN_PRUNE_JOBS"=>"Begin pruning Jobs older than %{INT} month %{INT} days ."}
2019-03-15T15:49:08.265088+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_BEGIN_PRUNE_FILES"=>"Begin pruning Files."}
2019-03-15T15:49:08.265224+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_PRUNED_JOBS"=>"Pruned %{INT} Jobs* for client %{BACULA_HOST:client} from catalog."}
2019-03-15T15:49:08.265349+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_PRUNED_FILES"=>"Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:client} from catalog."}
2019-03-15T15:49:08.265490+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_ENDPRUNE"=>"End auto prune."}
2019-03-15T15:49:08.265639+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_STARTJOB"=>"Start Backup JobId %{INT}, Job=%{BACULA_JOB:job}"}
2019-03-15T15:49:08.265772+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_STARTRESTORE"=>"Start Restore Job %{BACULA_JOB:job}"}
2019-03-15T15:49:08.265908+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_USEDEVICE"=>"Using Device \\\"%{BACULA_DEVICE:device}\\\""}
2019-03-15T15:49:08.266030+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.265 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_DIFF_FS"=>"\\s+%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it."}
2019-03-15T15:49:08.266149+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_JOBEND"=>"Job write elapsed time = %{DATA:elapsed}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second"}
2019-03-15T15:49:08.266261+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOPRUNE_JOBS"=>"No Jobs found to prune."}
2019-03-15T15:49:08.266372+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOPRUNE_FILES"=>"No Files found to prune."}
2019-03-15T15:49:08.266489+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_VOLUME_PREVWRITTEN"=>"Volume \\\"%{BACULA_VOLUME:volume}\\\" previously written, moving to end of data."}
2019-03-15T15:49:08.266608+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_READYAPPEND"=>"Ready to append to end of Volume \\\"%{BACULA_VOLUME:volume}\\\" size=%{INT}"}
2019-03-15T15:49:08.266729+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_CANCELLING"=>"Cancelling duplicate JobId=%{INT}."}
2019-03-15T15:49:08.266846+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_MARKCANCEL"=>"JobId %{INT}, Job %{BACULA_JOB:job} marked to be canceled."}
2019-03-15T15:49:08.266958+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.266 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_CLIENT_RBJ"=>"shell command: run ClientRunBeforeJob \\\"%{GREEDYDATA:runjob}\\\""}
2019-03-15T15:49:08.267073+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_VSS"=>"(Generate )?VSS (Writer)?"}
2019-03-15T15:49:08.267194+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_MAXSTART"=>"Fatal error: Job canceled because max start delay time exceeded."}
2019-03-15T15:49:08.267309+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_DUPLICATE"=>"Fatal error: JobId %{INT:duplicate} already running. Duplicate job not allowed."}
2019-03-15T15:49:08.267421+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOJOBSTAT"=>"Fatal error: No Job status returned from FD."}
2019-03-15T15:49:08.267549+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_FATAL_CONN"=>"Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})"}
2019-03-15T15:49:08.267675+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NO_CONNECT"=>"Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?<berror>%{GREEDYDATA})"}
2019-03-15T15:49:08.267789+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NO_AUTH"=>"Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes:"}
2019-03-15T15:49:08.267903+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOSUIT"=>"No prior or suitable Full backup found in catalog. Doing FULL backup."}
2019-03-15T15:49:08.268009+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.267 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_NOPRIOR"=>"No prior Full backup Job record found."}
2019-03-15T15:49:08.268135+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.268 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOG_JOB"=>"(Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \\(%{BACULA_VERSION}\\):"}
2019-03-15T15:49:08.268504+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.268 [[main]-pipeline-manager] grok - Adding pattern {"BACULA_LOGLINE"=>"%{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})"}
2019-03-15T15:49:08.296773+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.296 [[main]-pipeline-manager] grok - replacement_pattern => (?<SYSLOG5424PRI:pri><%{NONNEGINT:syslog5424_pri}>)
2019-03-15T15:49:08.297001+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.296 [[main]-pipeline-manager] grok - replacement_pattern => (?<NONNEGINT:syslog5424_pri>\b(?:[0-9]+)\b)
2019-03-15T15:49:08.297236+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.297 [[main]-pipeline-manager] grok - replacement_pattern => (?<NUMBER:rfc_version>(?:%{BASE10NUM}))
2019-03-15T15:49:08.298589+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.298 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))))
2019-03-15T15:49:08.298783+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.298 [[main]-pipeline-manager] grok - replacement_pattern => (?<TIMESTAMP_ISO8601:timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?)
2019-03-15T15:49:08.298939+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.298 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?>\d\d){1,2})
2019-03-15T15:49:08.299112+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:0?[1-9]|1[0-2]))
2019-03-15T15:49:08.299254+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]))
2019-03-15T15:49:08.299436+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:2[0123]|[01]?[0-9]))
2019-03-15T15:49:08.299644+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:[0-5][0-9]))
2019-03-15T15:49:08.299802+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))
2019-03-15T15:49:08.299980+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.299 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:Z|[+-]%{HOUR}(?::?%{MINUTE})))
2019-03-15T15:49:08.300101+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:2[0123]|[01]?[0-9]))
2019-03-15T15:49:08.300224+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?:(?:[0-5][0-9]))
2019-03-15T15:49:08.300365+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?<USERNAME:app>[a-zA-Z0-9._-]+)
2019-03-15T15:49:08.300510+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?<USERNAME:dyno>[a-zA-Z0-9._-]+)
2019-03-15T15:49:08.300650+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?<USERNAME:proc>[a-zA-Z0-9._-]+)
2019-03-15T15:49:08.300852+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?<USERNAME:event_id>[a-zA-Z0-9._-]+)
2019-03-15T15:49:08.300993+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.300 [[main]-pipeline-manager] grok - replacement_pattern => (?<GREEDYDATA:message>.*)
2019-03-15T15:49:08.305498+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.305 [[main]-pipeline-manager] grok - Grok compiled OK {:pattern=>"%{SYSLOG5424PRI:pri}%{NUMBER:rfc_version} %{TIMESTAMP_ISO8601:timestamp} %{USERNAME:app} %{USERNAME:dyno} %{USERNAME:proc} - Event %{USERNAME:event_id}: %{GREEDYDATA:message}", :expanded_pattern=>"(?<SYSLOG5424PRI:pri><(?<NONNEGINT:syslog5424_pri>\\b(?:[0-9]+)\\b)>)(?<NUMBER:rfc_version>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))))) (?<TIMESTAMP_ISO8601:timestamp>(?:(?>\\d\\d){1,2})-(?:(?:0?[1-9]|1[0-2]))-(?:(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]))[T ](?:(?:2[0123]|[01]?[0-9])):?(?:(?:[0-5][0-9]))(?::?(?:(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)))?(?:(?:Z|[+-](?:(?:2[0123]|[01]?[0-9]))(?::?(?:(?:[0-5][0-9])))))?) (?<USERNAME:app>[a-zA-Z0-9._-]+) (?<USERNAME:dyno>[a-zA-Z0-9._-]+) (?<USERNAME:proc>[a-zA-Z0-9._-]+) - Event (?<USERNAME:event_id>[a-zA-Z0-9._-]+): (?<GREEDYDATA:message>.*)"}
2019-03-15T15:49:08.400079+00:00 app[web.1]: [INFO ] 2019-03-15 15:49:08.399 [[main]-pipeline-manager] tcp - Starting tcp input listener {:address=>"0.0.0.0:48857", :ssl_enable=>"false"}
2019-03-15T15:49:08.559488+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.556 [[main]-pipeline-manager] InternalLoggerFactory - Using SLF4J as the default logging framework
2019-03-15T15:49:08.563381+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.563 [[main]-pipeline-manager] MultithreadEventLoopGroup - -Dio.netty.eventLoopThreads: 4
2019-03-15T15:49:08.592429+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.592 [[main]-pipeline-manager] PlatformDependent0 - -Dio.netty.noUnsafe: false
2019-03-15T15:49:08.592699+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.592 [[main]-pipeline-manager] PlatformDependent0 - Java version: 8
2019-03-15T15:49:08.593767+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.593 [[main]-pipeline-manager] PlatformDependent0 - sun.misc.Unsafe.theUnsafe: available
2019-03-15T15:49:08.594233+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.594 [[main]-pipeline-manager] PlatformDependent0 - sun.misc.Unsafe.copyMemory: available
2019-03-15T15:49:08.594776+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.594 [[main]-pipeline-manager] PlatformDependent0 - java.nio.Buffer.address: available
2019-03-15T15:49:08.595321+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.595 [[main]-pipeline-manager] PlatformDependent0 - direct buffer constructor: available
2019-03-15T15:49:08.596262+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent0 - java.nio.Bits.unaligned: available, true
2019-03-15T15:49:08.596327+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent0 - jdk.internal.misc.Unsafe.allocateUninitializedArray(int): unavailable prior to Java9
2019-03-15T15:49:08.596357+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent0 - java.nio.DirectByteBuffer.<init>(long, int): available
2019-03-15T15:49:08.596419+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent - sun.misc.Unsafe: available
2019-03-15T15:49:08.596866+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent - -Dio.netty.tmpdir: /tmp (java.io.tmpdir)
2019-03-15T15:49:08.596960+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.596 [[main]-pipeline-manager] PlatformDependent - -Dio.netty.bitMode: 64 (sun.arch.data.model)
2019-03-15T15:49:08.597811+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.597 [[main]-pipeline-manager] PlatformDependent - -Dio.netty.noPreferDirect: false
2019-03-15T15:49:08.597928+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.597 [[main]-pipeline-manager] PlatformDependent - -Dio.netty.maxDirectMemory: 1056309248 bytes
2019-03-15T15:49:08.598029+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.597 [[main]-pipeline-manager] PlatformDependent - -Dio.netty.uninitializedArrayAllocationThreshold: -1
2019-03-15T15:49:08.598990+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.598 [[main]-pipeline-manager] CleanerJava6 - java.nio.ByteBuffer.cleaner(): available
2019-03-15T15:49:08.620308+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.620 [[main]-pipeline-manager] NioEventLoop - -Dio.netty.noKeySetOptimization: false
2019-03-15T15:49:08.620401+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.620 [[main]-pipeline-manager] NioEventLoop - -Dio.netty.selectorAutoRebuildThreshold: 512
2019-03-15T15:49:08.629895+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.629 [[main]-pipeline-manager] PlatformDependent - org.jctools-core.MpscChunkedArrayQueue: available
2019-03-15T15:49:08.658441+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.658 [[main]-pipeline-manager] DefaultChannelId - -Dio.netty.processId: 12 (auto-detected)
2019-03-15T15:49:08.661384+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.661 [[main]-pipeline-manager] NetUtil - -Djava.net.preferIPv4Stack: true
2019-03-15T15:49:08.661427+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.661 [[main]-pipeline-manager] NetUtil - -Djava.net.preferIPv6Addresses: false
2019-03-15T15:49:08.663233+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.663 [[main]-pipeline-manager] NetUtil - Loopback interface: lo (lo, 127.0.0.1)
2019-03-15T15:49:08.663986+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.663 [[main]-pipeline-manager] NetUtil - /proc/sys/net/core/somaxconn: 65535
2019-03-15T15:49:08.665068+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.664 [[main]-pipeline-manager] DefaultChannelId - -Dio.netty.machineId: 12:53:92:ff:fe:35:e2:92 (auto-detected)
2019-03-15T15:49:08.671211+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.671 [[main]-pipeline-manager] InternalThreadLocalMap - -Dio.netty.threadLocalMap.stringBuilder.initialSize: 1024
2019-03-15T15:49:08.671251+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.671 [[main]-pipeline-manager] InternalThreadLocalMap - -Dio.netty.threadLocalMap.stringBuilder.maxSize: 4096
2019-03-15T15:49:08.681939+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.681 [[main]-pipeline-manager] ResourceLeakDetector - -Dio.netty.leakDetection.level: simple
2019-03-15T15:49:08.681976+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.681 [[main]-pipeline-manager] ResourceLeakDetector - -Dio.netty.leakDetection.targetRecords: 4
2019-03-15T15:49:08.708901+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.708 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.numHeapArenas: 4
2019-03-15T15:49:08.708941+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.708 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.numDirectArenas: 4
2019-03-15T15:49:08.708942+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.708 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.pageSize: 8192
2019-03-15T15:49:08.708965+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.708 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.maxOrder: 11
2019-03-15T15:49:08.709028+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.708 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.chunkSize: 16777216
2019-03-15T15:49:08.709061+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.tinyCacheSize: 512
2019-03-15T15:49:08.709110+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.smallCacheSize: 256
2019-03-15T15:49:08.709157+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.normalCacheSize: 64
2019-03-15T15:49:08.709225+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.maxCachedBufferCapacity: 32768
2019-03-15T15:49:08.709252+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.cacheTrimInterval: 8192
2019-03-15T15:49:08.709312+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.709 [[main]-pipeline-manager] PooledByteBufAllocator - -Dio.netty.allocator.useCacheForAllThreads: true
2019-03-15T15:49:08.718741+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.718 [[main]-pipeline-manager] ByteBufUtil - -Dio.netty.allocator.type: pooled
2019-03-15T15:49:08.718827+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.718 [[main]-pipeline-manager] ByteBufUtil - -Dio.netty.threadLocalDirectBufferSize: 65536
2019-03-15T15:49:08.718878+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.718 [[main]-pipeline-manager] ByteBufUtil - -Dio.netty.maxThreadLocalCharBufferSize: 16384
2019-03-15T15:49:08.766748+00:00 app[web.1]: [INFO ] 2019-03-15 15:49:08.766 [Converge PipelineAction::Create<main>] pipeline - Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x35c2cfbc run>"}
2019-03-15T15:49:08.847818+00:00 app[web.1]: [INFO ] 2019-03-15 15:49:08.847 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
2019-03-15T15:49:08.897507+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.897 [Api Webserver] agent - Starting puma
2019-03-15T15:49:08.907787+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.907 [Api Webserver] agent - Trying to start WebServer {:port=>9600}
2019-03-15T15:49:09.117844+00:00 heroku[web.1]: State changed from starting to up
2019-03-15T15:49:08.960288+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:08.960 [Api Webserver] service - [api-service] start
2019-03-15T15:49:09.032215+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.031 [nioEventLoopGroup-2-1] line - config LogStash::Codecs::Line/@id = "line_a169b908-c5b8-461e-bc74-4677b03ede73"
2019-03-15T15:49:09.032569+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.032 [nioEventLoopGroup-2-1] line - config LogStash::Codecs::Line/@enable_metric = true
2019-03-15T15:49:09.032714+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.032 [nioEventLoopGroup-2-1] line - config LogStash::Codecs::Line/@charset = "UTF-8"
2019-03-15T15:49:09.033036+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.032 [nioEventLoopGroup-2-1] line - config LogStash::Codecs::Line/@delimiter = "\n"
2019-03-15T15:49:09.057134+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.056 [nioEventLoopGroup-2-1] Recycler - -Dio.netty.recycler.maxCapacityPerThread: 32768
2019-03-15T15:49:09.057248+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.057 [nioEventLoopGroup-2-1] Recycler - -Dio.netty.recycler.maxSharedCapacityFactor: 2
2019-03-15T15:49:09.057344+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.057 [nioEventLoopGroup-2-1] Recycler - -Dio.netty.recycler.linkCapacity: 16
2019-03-15T15:49:09.057424+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.057 [nioEventLoopGroup-2-1] Recycler - -Dio.netty.recycler.ratio: 8
2019-03-15T15:49:09.071781+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.071 [nioEventLoopGroup-2-1] AbstractByteBuf - -Dio.netty.buffer.bytebuf.checkAccessible: true
2019-03-15T15:49:09.078297+00:00 app[web.1]: [DEBUG] 2019-03-15 15:49:09.078 [nioEventLoopGroup-2-1] ResourceLeakDetectorFactory - Loaded default ResourceLeakDetector: io.netty.util.ResourceLeakDetector@369aecb3
2019-03-15T15:49:09.199984+00:00 app[web.1]: [INFO ] 2019-03-15 15:49:09.199 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment