mmmmmmpc/gist:c38c8d6ddea612821621cd6d09c3f846

## gistfile1.txt
+ fluentdargs=-vv
+ echo '>>>>>> ENVIRONMENT VARS <<<<<'
+ env
+ sort
>>>>>> ENVIRONMENT VARS <<<<<
DATA_VERSION=1.6.0
ES_CA=/etc/fluent/keys/ca
ES_CLIENT_CERT=/etc/fluent/keys/cert
ES_CLIENT_KEY=/etc/fluent/keys/key
ES_COPY=false
ES_COPY_CA=
ES_COPY_CLIENT_CERT=
ES_COPY_CLIENT_KEY=
ES_COPY_HOST=
ES_COPY_PASSWORD=
ES_COPY_PORT=
ES_COPY_SCHEME=https
ES_COPY_USERNAME=
ES_HOST=logging-es
ES_PORT=9200
FLUENTD_VERSION=0.12.39
GEM_HOME=/opt/app-root/src
HOME=/opt/app-root/src
HOSTNAME=logging-fluentd-3p494
JOURNAL_READ_FROM_HEAD=
JOURNAL_SOURCE=
K8S_HOST_URL=https://kubernetes.default.svc.cluster.local
KUBERNETES_PORT=tcp://172.30.0.1:443
KUBERNETES_PORT_443_TCP=tcp://172.30.0.1:443
KUBERNETES_PORT_443_TCP_ADDR=172.30.0.1
KUBERNETES_PORT_443_TCP_PORT=443
KUBERNETES_PORT_443_TCP_PROTO=tcp
KUBERNETES_PORT_53_TCP=tcp://172.30.0.1:53
KUBERNETES_PORT_53_TCP_ADDR=172.30.0.1
KUBERNETES_PORT_53_TCP_PORT=53
KUBERNETES_PORT_53_TCP_PROTO=tcp
KUBERNETES_PORT_53_UDP=udp://172.30.0.1:53
KUBERNETES_PORT_53_UDP_ADDR=172.30.0.1
KUBERNETES_PORT_53_UDP_PORT=53
KUBERNETES_PORT_53_UDP_PROTO=udp
KUBERNETES_SERVICE_HOST=172.30.0.1
KUBERNETES_SERVICE_PORT=443
KUBERNETES_SERVICE_PORT_DNS=53
KUBERNETES_SERVICE_PORT_DNS_TCP=53
KUBERNETES_SERVICE_PORT_HTTPS=443
LOGGING_ES_CLUSTER_PORT=tcp://172.30.46.53:9300
LOGGING_ES_CLUSTER_PORT_9300_TCP=tcp://172.30.46.53:9300
LOGGING_ES_CLUSTER_PORT_9300_TCP_ADDR=172.30.46.53
LOGGING_ES_CLUSTER_PORT_9300_TCP_PORT=9300
LOGGING_ES_CLUSTER_PORT_9300_TCP_PROTO=tcp
LOGGING_ES_CLUSTER_SERVICE_HOST=172.30.46.53
LOGGING_ES_CLUSTER_SERVICE_PORT=9300
LOGGING_ES_PORT=tcp://172.30.215.97:9200
LOGGING_ES_PORT_9200_TCP=tcp://172.30.215.97:9200
LOGGING_ES_PORT_9200_TCP_ADDR=172.30.215.97
LOGGING_ES_PORT_9200_TCP_PORT=9200
LOGGING_ES_PORT_9200_TCP_PROTO=tcp
LOGGING_ES_SERVICE_HOST=172.30.215.97
LOGGING_ES_SERVICE_PORT=9200
LOGGING_KIBANA_PORT=tcp://172.30.174.26:443
LOGGING_KIBANA_PORT_443_TCP=tcp://172.30.174.26:443
LOGGING_KIBANA_PORT_443_TCP_ADDR=172.30.174.26
LOGGING_KIBANA_PORT_443_TCP_PORT=443
LOGGING_KIBANA_PORT_443_TCP_PROTO=tcp
LOGGING_KIBANA_SERVICE_HOST=172.30.174.26
LOGGING_KIBANA_SERVICE_PORT=443
OPS_CA=/etc/fluent/keys/ca
OPS_CLIENT_CERT=/etc/fluent/keys/cert
OPS_CLIENT_KEY=/etc/fluent/keys/key
OPS_COPY_CA=
OPS_COPY_CLIENT_CERT=
OPS_COPY_CLIENT_KEY=
OPS_COPY_HOST=
OPS_COPY_PASSWORD=
OPS_COPY_PORT=
OPS_COPY_SCHEME=https
OPS_COPY_USERNAME=
OPS_HOST=logging-es
OPS_PORT=9200
PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
PWD=/opt/app-root/src
RUBY_VERSION=2.0
SHLVL=1
USE_JOURNAL=
VERBOSE=true
_=/usr/bin/env
>>>>>>>>>>>>><<<<<<<<<<<<<<<<
+ echo '>>>>>>>>>>>>><<<<<<<<<<<<<<<<'
+ '[' -z '' -o '' = false ']'
+ '[' -z '' -o '' = true ']'
+ '[' -z '' ']'
+ '[' -d /var/log/journal ']'
+ export JOURNAL_SOURCE=/run/log/journal
+ JOURNAL_SOURCE=/run/log/journal
+ '[' -z '' ']'
+ docker_uses_journal
+ grep -q '^[^#].*"log-driver":' /etc/docker/daemon.json
+ grep -q '^OPTIONS='\''[^'\'']*--log-driver=journald' /etc/sysconfig/docker
+ return 0
+ export USE_JOURNAL=true
+ USE_JOURNAL=true
++ /usr/sbin/ip -4 addr show dev eth0
++ grep inet
++ sed -e 's/[ \t]*inet \([0-9.]*\).*/\1/'
+ IPADDR4=10.128.0.58
++ grep inet6
++ /usr/sbin/ip -6 addr show dev eth0
++ sed 's/[ \t]*inet6 \([a-f0-9:]*\).*/\1/'
+ IPADDR6=fe80::4476:fff:fe82:c3d0
+ export IPADDR4 IPADDR6
+ BUFFER_SIZE_LIMIT=16777216
+ CFG_DIR=/etc/fluent/configs.d
+ '[' '' = true ']'
+ ruby generate_throttle_configs.rb
+ rm -f '/etc/fluent/configs.d/openshift/*mux*.conf'
+ '[' true = true ']'
+ export BUFFER_QUEUE_FULL_ACTION=block
+ BUFFER_QUEUE_FULL_ACTION=block
+ K8S_FILTER_REMOVE_KEYS=log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+ '[' -n '' ']'
+ rm -f /etc/fluent/configs.d/openshift/filter-pre-mux-client.conf /etc/fluent/configs.d/openshift/output-pre-mux-client.conf
+ export K8S_FILTER_REMOVE_KEYS
+ '[' -n '' ']'
+ NUM_OUTPUTS=2
+ '[' false = true ']'
+ FILE_BUFFER_PATH=/var/lib/fluentd
+ mkdir -p /var/lib/fluentd
++ df -B1 /var/lib/fluentd
++ grep -v Filesystem
++ awk '{print $2}'
+ DF_LIMIT=10718543872
+ DF_LIMIT=10718543872
++ expr 10718543872 / 4
+ DF_LIMIT=2679635968
+ '[' 2679635968 -eq 0 ']'
++ echo 2Gi
++ sed -e 's/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g'
++ bc
+ TOTAL_LIMIT=2147483648
+ '[' 2147483648 -le 0 ']'
++ expr 2147483648 '*' 2
WARNING: Available disk space (2679635968 bytes) is less than the user specified file buffer limit ( times 2).
+ TOTAL_LIMIT=4294967296
+ '[' 2679635968 -lt 4294967296 ']'
+ echo 'WARNING: Available disk space (2679635968 bytes) is less than the user specified file buffer limit ( times 2).'
+ TOTAL_LIMIT=2679635968
++ echo 16777216
++ sed -e 's/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g'
++ bc
+ BUFFER_SIZE_LIMIT=16777216
+ BUFFER_SIZE_LIMIT=16777216
++ expr 2679635968 / 2
+ TOTAL_BUFFER_SIZE_LIMIT=1339817984
+ '[' -z 1339817984 -o 1339817984 -eq 0 ']'
++ expr 1339817984 / 16777216
+ BUFFER_QUEUE_LIMIT=79
+ '[' -z 79 -o 79 -eq 0 ']'
+ export BUFFER_QUEUE_LIMIT BUFFER_SIZE_LIMIT
+ OPS_COPY_HOST=
+ OPS_COPY_PORT=
+ OPS_COPY_SCHEME=https
+ OPS_COPY_CLIENT_CERT=
+ OPS_COPY_CLIENT_KEY=
+ OPS_COPY_CA=
+ OPS_COPY_USERNAME=
+ OPS_COPY_PASSWORD=
+ export OPS_COPY_HOST OPS_COPY_PORT OPS_COPY_SCHEME OPS_COPY_CLIENT_CERT OPS_COPY_CLIENT_KEY OPS_COPY_CA OPS_COPY_USERNAME OPS_COPY_PASSWORD
+ '[' false = true -a -z '' ']'
+ '[' false = true ']'
+ echo
+ echo
+ '[' '' = true ']'
+ rm -f /etc/fluent/configs.d/openshift/input-pre-monitor.conf
+ '[' '' = true ']'
+ rm -f /etc/fluent/configs.d/openshift/input-pre-debug.conf
+ '[' '' = true ']'
+ echo 'umounts of dead containers will fail. Ignoring...'
umounts of dead containers will fail. Ignoring...
+ umount /var/lib/docker/containers/094861ad7d1ff01800740c2a26dee27b7390ff2f65c99c6fcd86c878f5774c3a/shm /var/lib/docker/containers/14caecbc2a842e170cbf258f8729fa61271b665718d2636b33d0dc48d1922119/shm /var/lib/docker/containers/30dfb535745ee538e1f369921c5c0a58ce585a0cf1071a2f402eaa77c9eae322/shm /var/lib/docker/containers/3f8ef8e0b5c94930e658e7887a382c53e708e0710739a465d975d03904c1fd37/shm /var/lib/docker/containers/40147a4d04439d57cf3d5dbe9392ff17f19ef84419b01e196b8d22177c607e35/shm /var/lib/docker/containers/51d91bb11b27fd1194589252ea91502f1d1b84778473601ec03dba9e82c4e875/shm /var/lib/docker/containers/5790c63ca773ae1b6570f679c76dc13633f1cc9056c7d8476ae5b4d7872e473f/shm /var/lib/docker/containers/83a8afb4ad1a317bf15c2c6b5fd5ff233207a53d6786fb4c008f6e59efa85b01/shm /var/lib/docker/containers/8ee7f26fe6206640c9256d4d75a14263f41d5e9736e6be1491d66d6fccb32e2d/shm /var/lib/docker/containers/971a12553830c82dd686e8b7a7fd3c5b4e2b92ca8653090b8b8e37fbbd3cc265/shm /var/lib/docker/containers/a9a19a76a54c7049fec0ee1889736442ff34f6a75b4087695a27d92916f5ea59/shm /var/lib/docker/containers/c39c38c55afb502309c50e99133a5e68810a5ede6c46c613d5858ef6b2540c06/shm /var/lib/docker/containers/cd829147111082528352c03eef385c78c2f2547061791075504f052603e32666/shm /var/lib/docker/containers/d2c1d36a5aa1ee51a3b695d25f729017e96d0605e52ecd092e29ddc3f1488550/shm
umount: /var/lib/docker/containers/14caecbc2a842e170cbf258f8729fa61271b665718d2636b33d0dc48d1922119/shm: not mounted
umount: /var/lib/docker/containers/40147a4d04439d57cf3d5dbe9392ff17f19ef84419b01e196b8d22177c607e35/shm: not mounted
umount: /var/lib/docker/containers/51d91bb11b27fd1194589252ea91502f1d1b84778473601ec03dba9e82c4e875/shm: not mounted
umount: /var/lib/docker/containers/5790c63ca773ae1b6570f679c76dc13633f1cc9056c7d8476ae5b4d7872e473f/shm: not mounted
umount: /var/lib/docker/containers/8ee7f26fe6206640c9256d4d75a14263f41d5e9736e6be1491d66d6fccb32e2d/shm: not mounted
umount: /var/lib/docker/containers/971a12553830c82dd686e8b7a7fd3c5b4e2b92ca8653090b8b8e37fbbd3cc265/shm: not mounted
umount: /var/lib/docker/containers/d2c1d36a5aa1ee51a3b695d25f729017e96d0605e52ecd092e29ddc3f1488550/shm: not mounted
+ :
+ [[ -n '' ]]
+ exec fluentd -vv
2017-08-25 14:08:10 +0000 [info]: fluent/supervisor.rb:471:read_config: reading config file path="/etc/fluent/fluent.conf"
2017-08-25 14:08:15 +0000 [info]: fluent/supervisor.rb:337:supervise: starting fluentd-0.12.39
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered buffer plugin 'file'
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered buffer plugin 'memory'
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'grep'
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'parser'
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'record_transformer'
2017-08-25 14:08:15 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'stdout'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'debug_agent'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'dummy'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'exec'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'forward'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'gc_stat'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'http'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'monitor_agent'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'object_space'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'status'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'unix'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'syslog'
2017-08-25 14:08:16 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'tail'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'tcp'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'udp'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'copy'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'exec'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'exec_filter'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'file'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'forward'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'null'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'relabel'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'roundrobin'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'stdout'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'tcp'
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'unix'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-elasticsearch' version '1.9.5'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-kubernetes_metadata_filter' version '0.27.0'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-rewrite-tag-filter' version '1.5.6'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-secure-forward' version '0.4.5'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-systemd' version '0.0.8'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluent-plugin-viaq_data_model' version '0.0.5'
2017-08-25 14:08:17 +0000 [info]: fluent/engine.rb:126:block in configure: gem 'fluentd' version '0.12.39'
2017-08-25 14:08:17 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="journal" type="grep"
2017-08-25 14:08:17 +0000 [warn]: config/section.rb:124:block in generate: 'exclude1' parameter is deprecated: Use <exclude> section
2017-08-25 14:08:17 +0000 [info]: fluent/agent.rb:129:add_match: adding match in @INGRESS pattern="journal" type="rewrite_tag_filter"
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'rewrite_tag_filter'
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule1 ["CONTAINER_NAME", /^k8s_kibana\./, "", "kubernetes.journal.container.kibana"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule2 ["CONTAINER_NAME", /^k8s_[^\.]+\.[^_]+_[^_]+_default_/, "", "kubernetes.journal.container._default_"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule3 ["CONTAINER_NAME", /^k8s_[^\.]+\.[^_]+_[^_]+_openshift-infra_/, "", "kubernetes.journal.container._openshift-infra_"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule4 ["CONTAINER_NAME", /^k8s_[^\.]+\.[^_]+_[^_]+_openshift_/, "", "kubernetes.journal.container._openshift_"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule5 ["CONTAINER_NAME", /^k8s_.*fluentd/, "", "kubernetes.journal.container.fluentd"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule6 ["CONTAINER_NAME", /^k8s_/, "", "kubernetes.journal.container"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule7 ["CONTAINER_NAME", /_openshift-infra_/, "", "journal.container._openshift-infra_"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule8 ["CONTAINER_NAME", /_openshift_/, "", "journal.container._openshift_"]
2017-08-25 14:08:17 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule9 ["_TRANSPORT", /.+/, "", "journal.system"]
2017-08-25 14:08:17 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="kubernetes.**" type="kubernetes_metadata"
2017-08-25 14:08:17 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'kubernetes_metadata'
2017-08-25 14:08:22 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="**kibana**" type="record_transformer"
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="kubernetes.journal.container**" type="record_transformer"
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="kubernetes.var.log.containers**" type="record_transformer"
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:129:add_match: adding match in @INGRESS pattern="journal.system**" type="rewrite_tag_filter"
2017-08-25 14:08:23 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule1 ["_SYSTEMD_UNIT", /^origin-master.service/, "", "journal.system.kubernetes.master"]
2017-08-25 14:08:23 +0000 [info]: plugin/out_rewrite_tag_filter.rb:55:block in configure: adding rewrite_tag_filter rule: rewriterule2 ["_SYSTEMD_UNIT", /^origin-node.service/, "", "journal.system.kubernetes.node"]
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="system.var.log**" type="record_transformer"
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="journal.system**" type="record_transformer"
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:141:add_filter: adding filter in @INGRESS pattern="**" type="viaq_data_model"
2017-08-25 14:08:23 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered filter plugin 'viaq_data_model'
2017-08-25 14:08:23 +0000 [info]: fluent/agent.rb:129:add_match: adding match in @INGRESS pattern="journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops" type="copy"
2017-08-25 14:08:23 +0000 [debug]: plugin/out_copy.rb:44:block in configure: adding store type="elasticsearch_dynamic"
2017-08-25 14:08:26 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'elasticsearch'
2017-08-25 14:08:26 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered output plugin 'elasticsearch_dynamic'
2017-08-25 14:08:26 +0000 [warn]: fluent/buffer.rb:164:configure: 'block' action stops input process until the buffer full is resolved. Check your pipeline this action is fit or not
2017-08-25 14:08:26 +0000 [info]: fluent/agent.rb:129:add_match: adding match in @INGRESS pattern="**" type="copy"
2017-08-25 14:08:26 +0000 [debug]: plugin/out_copy.rb:44:block in configure: adding store type="elasticsearch_dynamic"
2017-08-25 14:08:26 +0000 [warn]: fluent/buffer.rb:164:configure: 'block' action stops input process until the buffer full is resolved. Check your pipeline this action is fit or not
2017-08-25 14:08:26 +0000 [info]: fluent/root_agent.rb:152:add_source: adding source type="systemd"
2017-08-25 14:08:27 +0000 [trace]: fluent/plugin.rb:122:register_impl: registered input plugin 'systemd'
2017-08-25 14:08:27 +0000 [info]: fluent/engine.rb:133:configure: using configuration file: <ROOT>
  <source>
    @type systemd
    @label @INGRESS
    path /run/log/journal
    pos_file /var/log/journal.pos
    tag journal
  </source>
  <label @INGRESS>
    <filter journal>
      @type grep
      exclude1 PRIORITY ^7$
    </filter>
    <match journal>
      @type rewrite_tag_filter
      rewriterule1 CONTAINER_NAME ^k8s_kibana\. kubernetes.journal.container.kibana
      rewriterule2 CONTAINER_NAME ^k8s_[^\.]+\.[^_]+_[^_]+_default_ kubernetes.journal.container._default_
      rewriterule3 CONTAINER_NAME ^k8s_[^\.]+\.[^_]+_[^_]+_openshift-infra_ kubernetes.journal.container._openshift-infra_
      rewriterule4 CONTAINER_NAME ^k8s_[^\.]+\.[^_]+_[^_]+_openshift_ kubernetes.journal.container._openshift_
      rewriterule5 CONTAINER_NAME ^k8s_.*fluentd kubernetes.journal.container.fluentd
      rewriterule6 CONTAINER_NAME ^k8s_ kubernetes.journal.container
      rewriterule7 CONTAINER_NAME _openshift-infra_ journal.container._openshift-infra_
      rewriterule8 CONTAINER_NAME _openshift_ journal.container._openshift_
      rewriterule9 _TRANSPORT .+ journal.system
    </match>
    <filter kubernetes.**>
      type kubernetes_metadata
      kubernetes_url https://kubernetes.default.svc.cluster.local
      bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
      ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      include_namespace_id true
      use_journal true
      container_name_to_kubernetes_regexp ^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$
    </filter>
    <filter **kibana**>
      @type record_transformer
      enable_ruby
      remove_keys req,res,msg,name,level,v,pid,err
      <record>
        log ${(err rescue nil) || (msg rescue nil) || record['MESSAGE'] || log}
      </record>
    </filter>
    <filter kubernetes.journal.container**>
      @type record_transformer
      enable_ruby
      remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
      <record>
        hostname ${(record['kubernetes']['host'] rescue nil) || File.open('/etc/docker-hostname') { |f| f.readline }.rstrip}
        message ${record['message'] || record['MESSAGE'] || log}
        level ${["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][begin; ('%d' % record['PRIORITY'] || 9).to_i; rescue; 9; end]}
        time ${Time.at((record["_SOURCE_REALTIME_TIMESTAMP"] || record["__REALTIME_TIMESTAMP"]).to_f / 1000000.0).utc.to_datetime.rfc3339(6)}
        pipeline_metadata {"collector":{"ipaddr4":"${ENV['IPADDR4']}","ipaddr6":"${ENV['IPADDR6']}","inputname":"fluent-plugin-systemd","name":"fluentd openshift","received_at":"${Time.at((record['_SOURCE_REALTIME_TIMESTAMP'] || record['__REALTIME_TIMESTAMP']).to_f / 1000000.0).utc.to_datetime.rfc3339(6)}","version":"${ENV['FLUENTD_VERSION'] + ' ' + ENV['DATA_VERSION']}"}}
        systemd {"t":{"MACHINE_ID":"${record['_MACHINE_ID']}","AUDIT_LOGINUID":"${record['_AUDIT_LOGINUID']}","AUDIT_SESSION":"${record['_AUDIT_SESSION']}","BOOT_ID":"${record['_BOOT_ID']}","CAP_EFFECTIVE":"${record['_CAP_EFFECTIVE']}","CMDLINE":"${record['_CMDLINE']}","COMM":"${record['_COMM']}","EXE":"${record['_EXE']}","GID":"${record['_GID']}","HOSTNAME":"${record['_HOSTNAME']}","PID":"${record['_PID']}","SELINUX_CONTEXT":"${record['_SELINUX_CONTEXT']}","SOURCE_REALTIME_TIMESTAMP":"${record['_SOURCE_REALTIME_TIMESTAMP']}","SYSTEMD_CGROUP":"${record['_SYSTEMD_CGROUP']}","SYSTEMD_OWNER_UID":"${record['_SYSTEMD_OWNER_UID']}","SYSTEMD_SESSION":"${record['_SYSTEMD_SESSION']}","SYSTEMD_SLICE":"${record['_SYSTEMD_SLICE']}","SYSTEMD_UNIT":"${record['_SYSTEMD_UNIT']}","SYSTEMD_USER_UNIT":"${record['_SYSTEMD_USER_UNIT']}","TRANSPORT":"${record['_TRANSPORT']}","UID":"${record['_UID']}"},"u":{"CODE_FILE":"${record['CODE_FILE']}","CODE_FUNCTION":"${record['CODE_FUNCTION']}","CODE_LINE":"${record['CODE_LINE']}","ERRNO":"${record['ERRNO']}","MESSAGE_ID":"${record['MESSAGE_ID']}","RESULT":"${record['RESULT']}","UNIT":"${record['UNIT']}","SYSLOG_FACILITY":"${record['SYSLOG_FACILITY']}","SYSLOG_IDENTIFIER":"${record['SYSLOG_IDENTIFIER']}","SYSLOG_PID":"${record['SYSLOG_PID']}"},"k":{"KERNEL_DEVICE":"${record['_KERNEL_DEVICE']}","KERNEL_SUBSYSTEM":"${record['_KERNEL_SUBSYSTEM']}","UDEV_SYSNAME":"${record['_UDEV_SYSNAME']}","UDEV_DEVNODE":"${record['_UDEV_DEVNODE']}","UDEV_DEVLINK":"${record['_UDEV_DEVLINK']}"}}
      </record>
    </filter>
    <filter kubernetes.var.log.containers**>
      @type record_transformer
      enable_ruby
      remove_keys log,stream
      <record>
        hostname ${(record['kubernetes']['host'] rescue nil) || File.open('/etc/docker-hostname') { |f| f.readline }.rstrip}
        message ${(message rescue nil) || log}
        pipeline_metadata {"collector":{"ipaddr4":"${ENV['IPADDR4']}","ipaddr6":"${ENV['IPADDR6']}","inputname":"fluent-plugin-in_tail","name":"fluentd openshift","received_at":"${record['time'].to_s}","version":"${ENV['FLUENTD_VERSION'] + ' ' + ENV['DATA_VERSION']}"}}
        level ${record['stream'] == 'stdout' ? 'info' : 'err'}
        time ${time.utc.to_datetime.rfc3339(6)}
      </record>
    </filter>
    <match journal.system**>
      @type rewrite_tag_filter
      rewriterule1 _SYSTEMD_UNIT ^origin-master.service journal.system.kubernetes.master
      rewriterule2 _SYSTEMD_UNIT ^origin-node.service journal.system.kubernetes.node
    </match>
    <filter system.var.log**>
      @type record_transformer
      enable_ruby
      remove_keys host,pid,ident
      <record>
        systemd {"t":{"PID":"${record['pid']}"},"u":{"SYSLOG_IDENTIFIER":"${record['ident']}"}}
        hostname ${host.eql?('localhost') ? (begin; File.open('/etc/docker-hostname') { |f| f.readline }.rstrip; rescue; host; end) : host}
        time ${ (Time.at(time) > Time.now) ? (Time.new((time.year - 1), time.month, time.day, time.hour, time.min, time.sec, time.utc_offset).utc.to_datetime.rfc3339(6)) : (time.utc.to_datetime.rfc3339(6)) }
        pipeline_metadata {"collector":{"ipaddr4":"${ENV['IPADDR4']}","ipaddr6":"${ENV['IPADDR6']}","inputname":"fluent-plugin-in_tail","name":"fluentd openshift","received_at":"${(Time.at(time) > Time.now) ? (Time.new((time.year - 1), time.month, time.day, time.hour, time.min, time.sec, time.utc_offset).utc.to_datetime.rfc3339(6)) : (time.utc.to_datetime.rfc3339(6))}","version":"${ENV['FLUENTD_VERSION'] + ' ' + ENV['DATA_VERSION']}"}}
      </record>
    </filter>
    <filter journal.system**>
      @type record_transformer
      enable_ruby
      remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
      <record>
        systemd {"t":{"MACHINE_ID":"${record['_MACHINE_ID']}","AUDIT_LOGINUID":"${record['_AUDIT_LOGINUID']}","AUDIT_SESSION":"${record['_AUDIT_SESSION']}","BOOT_ID":"${record['_BOOT_ID']}","CAP_EFFECTIVE":"${record['_CAP_EFFECTIVE']}","CMDLINE":"${record['_CMDLINE']}","COMM":"${record['_COMM']}","EXE":"${record['_EXE']}","GID":"${record['_GID']}","HOSTNAME":"${record['_HOSTNAME']}","PID":"${record['_PID']}","SELINUX_CONTEXT":"${record['_SELINUX_CONTEXT']}","SOURCE_REALTIME_TIMESTAMP":"${record['_SOURCE_REALTIME_TIMESTAMP']}","SYSTEMD_CGROUP":"${record['_SYSTEMD_CGROUP']}","SYSTEMD_OWNER_UID":"${record['_SYSTEMD_OWNER_UID']}","SYSTEMD_SESSION":"${record['_SYSTEMD_SESSION']}","SYSTEMD_SLICE":"${record['_SYSTEMD_SLICE']}","SYSTEMD_UNIT":"${record['_SYSTEMD_UNIT']}","SYSTEMD_USER_UNIT":"${record['_SYSTEMD_USER_UNIT']}","TRANSPORT":"${record['_TRANSPORT']}","UID":"${record['_UID']}"},"u":{"CODE_FILE":"${record['CODE_FILE']}","CODE_FUNCTION":"${record['CODE_FUNCTION']}","CODE_LINE":"${record['CODE_LINE']}","ERRNO":"${record['ERRNO']}","MESSAGE_ID":"${record['MESSAGE_ID']}","RESULT":"${record['RESULT']}","UNIT":"${record['UNIT']}","SYSLOG_FACILITY":"${record['SYSLOG_FACILITY']}","SYSLOG_IDENTIFIER":"${record['SYSLOG_IDENTIFIER']}","SYSLOG_PID":"${record['SYSLOG_PID']}"},"k":{"KERNEL_DEVICE":"${record['_KERNEL_DEVICE']}","KERNEL_SUBSYSTEM":"${record['_KERNEL_SUBSYSTEM']}","UDEV_SYSNAME":"${record['_UDEV_SYSNAME']}","UDEV_DEVNODE":"${record['_UDEV_DEVNODE']}","UDEV_DEVLINK":"${record['_UDEV_DEVLINK']}"}}
        hostname ${_HOSTNAME.eql?('localhost') ? (begin; File.open('/etc/docker-hostname') { |f| f.readline }.rstrip; rescue; _HOSTNAME; end) : _HOSTNAME}
        message ${record["MESSAGE"]}
        pipeline_metadata {"collector":{"ipaddr4":"${ENV['IPADDR4']}","ipaddr6":"${ENV['IPADDR6']}","inputname":"fluent-plugin-systemd","name":"fluentd openshift","received_at":"${(record['_SOURCE_REALTIME_TIMESTAMP'] || record['__REALTIME_TIMESTAMP']) ? Time.at((record['_SOURCE_REALTIME_TIMESTAMP'] || record['__REALTIME_TIMESTAMP']).to_f / 1000000.0).utc.to_datetime.rfc3339(6) : time.utc.to_datetime.rfc3339(6)}","version":"${ENV['FLUENTD_VERSION'] + ' ' + ENV['DATA_VERSION']}"}}
        time ${(record["_SOURCE_REALTIME_TIMESTAMP"] || record["__REALTIME_TIMESTAMP"]) ? Time.at((record["_SOURCE_REALTIME_TIMESTAMP"] || record["__REALTIME_TIMESTAMP"]).to_f / 1000000.0).utc.to_datetime.rfc3339(6) : time.utc.to_datetime.rfc3339(6)}
        level ${["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][begin; ('%d' % record['PRIORITY'] || 9).to_i; rescue; 9; end]}
      </record>
    </filter>
    <filter **>
      @type viaq_data_model
      default_keep_fields CEE,docker,file,geoip,hostname,kubernetes,level,message,offset,pid,pipeline_metadata,rsyslog,service,systemd,tags,time,ovirt,collectd,tlog,aushape,namespace_name,namespace_uuid
      extra_keep_fields
      keep_empty_fields message
      use_undefined false
      undefined_name undefined
      rename_time true
      rename_time_if_missing false
      src_time_name time
      dest_time_name @timestamp
    </filter>
    <match journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops>
      @type copy
      <store>
        @type elasticsearch_dynamic
        host logging-es
        port 9200
        scheme https
        index_name .operations.${begin record['@timestamp'].nil? ? Time.at(time).getutc.strftime(@logstash_dateformat) : Time.parse(record['@timestamp']).getutc.strftime(@logstash_dateformat) rescue $log.error("record is missing time and @timestamp - record " + record.to_s) end}
        user fluentd
        password xxxxxx
        client_key /etc/fluent/keys/key
        client_cert /etc/fluent/keys/cert
        ca_file /etc/fluent/keys/ca
        type_name com.redhat.viaq.common
        reload_connections false
        reload_on_failure false
        flush_interval 5s
        max_retry_wait 300
        disable_retry_limit true
        buffer_type file
        buffer_path /var/lib/fluentd/buffer-output-es-ops-config
        buffer_queue_limit 79
        buffer_chunk_limit 16777216
        buffer_queue_full_action block
      </store>
    </match>
    <match **>
      @type copy
      <store>
        @type elasticsearch_dynamic
        host logging-es
        port 9200
        scheme https
        index_name project.${begin record['kubernetes']['namespace_name'] rescue $log.error("record is missing kubernetes.namespace_name - record " + record.to_s) end}.${begin record['kubernetes']['namespace_id'] rescue $log.error("record is missing kubernetes.namespace_id - record " + record.to_s) end}.${begin Time.parse(record['@timestamp']).getutc.strftime(@logstash_dateformat) rescue $log.error("record is missing @timestamp - record " + record.to_s) end}
        user fluentd
        password xxxxxx
        client_key /etc/fluent/keys/key
        client_cert /etc/fluent/keys/cert
        ca_file /etc/fluent/keys/ca
        type_name com.redhat.viaq.common
        reload_connections false
        reload_on_failure false
        flush_interval 5s
        max_retry_wait 300
        disable_retry_limit true
        buffer_type file
        buffer_path /var/lib/fluentd/buffer-output-es-config
        buffer_queue_limit 79
        buffer_chunk_limit 16777216
        buffer_queue_full_action block
      </store>
    </match>
  </label>
</ROOT>
2017-08-25 14:08:34 +0000 [info]: plugin/out_elasticsearch_dynamic.rb:64:client: Connection opened to Elasticsearch cluster => {:host=>"logging-es", :port=>9200, :scheme=>"https", :user=>"fluentd", :password=>"obfuscated"}
2017-08-25 14:08:35 +0000 [info]: plugin/out_elasticsearch_dynamic.rb:64:client: Connection opened to Elasticsearch cluster => {:host=>"logging-es", :port=>9200, :scheme=>"https", :user=>"fluentd", :password=>"obfuscated"}