Created
June 16, 2020 01:24
-
-
Save iameli/a6fde74a1e0d7b58df2b19eb12942a7f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: apps/v1 | |
kind: DaemonSet | |
metadata: | |
name: fluent-bit | |
namespace: default | |
labels: | |
k8s-app: fluent-bit-logging | |
spec: | |
selector: | |
matchLabels: | |
k8s-app: fluent-bit-logging | |
template: | |
metadata: | |
labels: | |
k8s-app: fluent-bit-logging | |
spec: | |
containers: | |
- name: fluent-bit | |
image: iameli/fluent-bit:latest | |
env: | |
- name: LIVEPEER_NAMESPACE | |
value: tenant-livepeer | |
- name: PAPERTRAIL_HOST | |
valueFrom: | |
secretKeyRef: | |
name: livepeer-papertrail | |
key: PAPERTRAIL_HOST | |
- name: PAPERTRAIL_PORT | |
valueFrom: | |
secretKeyRef: | |
name: livepeer-papertrail | |
key: PAPERTRAIL_PORT | |
imagePullPolicy: Always | |
ports: | |
- containerPort: 2020 | |
volumeMounts: | |
- name: varlog | |
mountPath: /var/log | |
- name: varlibdockercontainers | |
mountPath: /var/lib/docker/containers | |
readOnly: true | |
- name: fluent-bit-config | |
mountPath: /fluent-bit/etc/ | |
terminationGracePeriodSeconds: 10 | |
volumes: | |
- name: varlog | |
hostPath: | |
path: /var/log | |
- name: varlibdockercontainers | |
hostPath: | |
path: /var/lib/docker/containers | |
- name: fluent-bit-config | |
configMap: | |
name: fluent-bit-config | |
serviceAccountName: fluentd | |
--- | |
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: fluent-bit-config | |
namespace: default | |
labels: | |
k8s-app: fluent-bit | |
data: | |
# Configuration files: server, input, filters and output | |
# ====================================================== | |
fluent-bit.conf: | | |
[SERVICE] | |
Flush 1 | |
Log_Level info | |
Daemon off | |
Parsers_File parsers.conf | |
HTTP_Server On | |
HTTP_Listen 0.0.0.0 | |
HTTP_Port 2020 | |
@INCLUDE input-kubernetes.conf | |
@INCLUDE filter-kubernetes.conf | |
@INCLUDE output-syslog.conf | |
input-kubernetes.conf: | | |
[INPUT] | |
Name tail | |
Path /var/log/containers/*.log | |
Parser docker | |
Tag kube.<namespace_name>.<container_name>.<pod_name>.<docker_id>- | |
Tag_Regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$ | |
Refresh_Interval 5 | |
Mem_Buf_Limit 5MB | |
Skip_Long_Lines On | |
filter-kubernetes.conf: | | |
[FILTER] | |
Name kubernetes | |
Match kube.* | |
Kube_Tag_Prefix kube. | |
Regex_Parser k8s-custom-tag | |
Merge_Log On | |
K8S-Logging.Parser Off | |
K8S-Logging.Exclude Off | |
[FILTER] | |
name nest | |
match kube.* | |
operation lift | |
nested_under kubernetes | |
output-syslog.conf: | | |
[OUTPUT] | |
Name syslog | |
Match kube.${LIVEPEER_NAMESPACE}.* | |
Host ${PAPERTRAIL_HOST} | |
Port ${PAPERTRAIL_PORT} | |
Mode udp | |
syslog_Hostname_key pod_name | |
syslog_Appname_key container_name | |
syslog_message_key log | |
output-stdout.conf: | | |
[OUTPUT] | |
Name stdout | |
Match kube.${LIVEPEER_NAMESPACE}.* | |
parsers.conf: | | |
[PARSER] | |
Name k8s-custom-tag | |
Format regex | |
Regex ^(?<namespace_name>[^_]+)\.(?<container_name>.+)\.(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?<docker_id>[a-z0-9]{64})-$ | |
[PARSER] | |
Name docker | |
Format json | |
Time_Key time | |
Time_Format %Y-%m-%dT%H:%M:%S.%L | |
Time_Keep On |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Glad people are finding this useful, but FWIW we've migrated entirely from Papertrail to Loki, using Promtail instead of fluent-bit for aggregation. Finding it wayyy better