Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save karstengresch/d31fde9e81dc4d8732119adcb09bbea7 to your computer and use it in GitHub Desktop.
Save karstengresch/d31fde9e81dc4d8732119adcb09bbea7 to your computer and use it in GitHub Desktop.
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: fluentd
labels:
app: fluentd
data:
fluentd.conf: |
@include prometheus.conf
@include systemd.conf
@include kubernetes.conf
@include apiserver-audit.conf
# Prevent fluentd from handling records containing its own logs. Otherwise
# it can lead to an infinite loop, when error in sending one message generates
# another message which also fails to be sent and so on.
<match fluent.**>
type null
</match>
# Used for health checking
<source>
@type http
port 9880
bind 0.0.0.0
</source>
# Emits internal metrics to every minute, and also exposes them on port
# 24220. Useful for determining if an output plugin is retryring/erroring,
# or determining the buffer queue length.
<source>
@type monitor_agent
bind 0.0.0.0
port 24220
tag fluentd.monitor.metrics
</source>
@include elasticsearch-output.conf
prometheus.conf: |
# input plugin that is required to expose metrics by other prometheus
# plugins, such as the prometheus_monitor input below.
<source>
@type prometheus
bind 0.0.0.0
port 24231
metrics_path /metrics
</source>
# input plugin that collects metrics from MonitorAgent and exposes them
# as prometheus metrics
<source>
@type prometheus_monitor
# update the metrics every 5 seconds
interval 5
</source>
<source>
@type prometheus_output_monitor
interval 5
</source>
<source>
@type prometheus_tail_monitor
interval 5
</source>
systemd.conf: |
<source>
@type systemd
pos_file /var/log/fluentd-journald-systemd.pos
read_from_head true
strip_underscores true
tag systemd
</source>
<match systemd>
@type rewrite_tag_filter
rewriterule1 SYSTEMD_UNIT ^(.+).service$ systemd.$1
rewriterule2 SYSTEMD_UNIT !^(.+).service$ systemd.unmatched
</match>
<filter kubelet>
type parser
format kubernetes
reserve_data true
key_name MESSAGE
</filter>
<filter docker>
type parser
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
reserve_data true
key_name MESSAGE
suppress_parse_error_log true
ignore_key_not_exist true
</filter>
# Filter filter ssh logs since it's mostly bots trying to login
<filter systemd.**>
@type grep
exclude1 SYSTEMD_UNIT (sshd@.*\.service)
</filter>
kubernetes.conf: |
# Capture Kubernetes pod logs
# The kubelet creates symlinks that capture the pod name, namespace,
# container name & Docker container ID to the docker logs for pods in the
# /var/log/containers directory on the host.
<source>
type tail
path /var/log/containers/*.log
pos_file /var/log/fluentd-containers.log.pos
time_format %Y-%m-%dT%H:%M:%S.%NZ
tag kubernetes.*
format json
read_from_head true
</source>
# Query the API for extra metadata.
<filter kubernetes.**>
type kubernetes_metadata
# If the logs begin with '{' and end with '}' then it's JSON so merge
# the JSON log field into the log event
merge_json_log true
preserve_json_log true
</filter>
# rewrite_tag_filter does not support nested fields like
# kubernetes.container_name, so this exists to flatten the fields
# so we can use them in our rewrite_tag_filter
<filter kubernetes.**>
@type record_transformer
enable_ruby true
<record>
kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]}
</record>
</filter>
# retag based on the container name of the log message
<match kubernetes.**>
@type rewrite_tag_filter
rewriterule1 kubernetes_namespace_container_name ^(.+)$ kube.$1
</match>
# Remove the unnecessary field as the information is already available on
# other fields.
<filter kube.**>
@type record_transformer
remove_keys kubernetes_namespace_container_name
</filter>
<filter kube.kube-system.**>
type parser
format kubernetes
reserve_data true
key_name log
suppress_parse_error_log true
ignore_key_not_exist true
</filter>
apiserver-audit.conf: |
# Example:
# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="<self>" asgroups="<lookup>" namespace="default" uri="/api/v1/namespaces/default/pods"
# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200"
<source>
type tail
format multiline
multiline_flush_interval 5s
format_firstline /^\S+\s+AUDIT:/
# Fields must be explicitly captured by name to be parsed into the record.
# Fields may not always be present, and order may change, so this just looks
# for a list of key="\"quoted\" value" pairs separated by spaces.
# Unknown fields are ignored.
# Note: We can't separate query/response lines as format1/format2 because
# they don't always come one after the other for a given query.
# TODO: Maybe add a JSON output mode to audit log so we can get rid of this?
format1 /^(?<time>\S+) AUDIT:(?: (?:id="(?<id>(?:[^"\\]|\\.)*)"|ip="(?<ip>(?:[^"\\]|\\.)*)"|method="(?<method>(?:[^"\\]|\\.)*)"|user="(?<user>(?:[^"\\]|\\.)*)"|groups="(?<groups>(?:[^"\\]|\\.)*)"|as="(?<as>(?:[^"\\]|\\.)*)"|asgroups="(?<asgroups>(?:[^"\\]|\\.)*)"|namespace="(?<namespace>(?:[^"\\]|\\.)*)"|uri="(?<uri>(?:[^"\\]|\\.)*)"|response="(?<response>(?:[^"\\]|\\.)*)"|\w+="(?:[^"\\]|\\.)*"))*/
time_format %FT%T.%L%Z
path /var/log/kube-apiserver-audit.log
pos_file /var/log/kube-apiserver-audit.log.pos
tag kube-apiserver-audit
</source>
elasticsearch-output.conf: |
<match **>
type elasticsearch
log_level info
include_tag_key true
# Replace with the host/port to your elasticsearch cluster.
# This assumes a service `elasticsearch` exists in the default namespace
host elasticsearch.default.svc.cluster.local
port 9200
logstash_format true
buffer_chunk_limit 2M
buffer_queue_limit 32
flush_interval 5s
max_retry_wait 30
disable_retry_limit
num_threads 8
template_file /fluentd/etc/elasticsearch-template-es5x.json
template_name elasticsearch-template-es5x.json
</match>
elasticsearch-template-es5x.json: |
{
"template" : "logstash-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 1
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "norms" : false},
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword" }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date", "include_in_all": false },
"@version": { "type": "keyword", "include_in_all": false },
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: fluentd
namespace: fluentd
labels:
app: fluentd
spec:
minReadySeconds: 10
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
app: fluentd
spec:
containers:
- name: fluentd
image: quay.io/coreos/fluentd-kubernetes:v0.12-debian-elasticsearch
command: ["sh", "-c", "gem install fluent-plugin-rewrite-tag-filter fluent-plugin-prometheus && fluentd -c /fluentd/etc/fluentd.conf -p /fluentd/plugins"]
# command: ["fluentd", "-c", "/fluentd/etc/fluentd.conf", "-p", "/fluentd/plugins"]
resources:
limits:
memory: 200Mi
requests:
cpu: 500m
memory: 200Mi
ports:
- name: prom-metrics
containerPort: 24231
protocol: TCP
- name: monitor-agent
containerPort: 24220
protocol: TCP
- name: http-input
containerPort: 9880
protocol: TCP
livenessProbe:
httpGet:
# Use percent encoding for query param.
# The value is {"log": "health check"}.
# the endpoint itself results in a new fluentd
# tag 'fluentd.pod-healthcheck'
path: /fluentd.pod.healthcheck?json=%7B%22log%22%3A+%22health+check%22%7D
port: 9880
initialDelaySeconds: 5
timeoutSeconds: 1
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: fluentd-config
mountPath: /fluentd/etc
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: fluentd-config
configMap:
name: "fluentd-config"
terminationGracePeriodSeconds: 30
serviceAccountName: fluentd
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
---
apiVersion: v1
kind: Namespace
metadata:
name: fluentd
---
apiVersion: monitoring.coreos.com/v1alpha1
kind: ServiceMonitor
metadata:
name: fluentd
namespace: fluentd
labels:
app: fluentd
spec:
jobLabel: app
selector:
matchLabels:
app: fluentd
namespaceSelector:
any: false
matchNames:
- fluentd
endpoints:
- port: prometheus-metrics
interval: 15s
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: fluentd-read
namespace: fluentd
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fluentd-read
subjects:
- kind: ServiceAccount
name: fluentd
namespace: fluentd
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: fluentd-read
namespace: fluentd
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs: ["get", "list", "watch"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: fluentd
---
apiVersion: v1
kind: Service
metadata:
name: fluentd
namespace: fluentd
labels:
app: fluentd
spec:
type: ClusterIP
clusterIP: None
selector:
app: fluentd
ports:
# Exposes Prometheus metrics
- name: prometheus-metrics
port: 24231
targetPort: prom-metrics
protocol: TCP
# Can be accessed using "kubectl proxy" at:
# http://127.0.0.1:8001/api/v1/proxy/namespaces/kube-system/services/fluentd:monitor-agent/api/plugins.json
- name: monitor-agent
port: 24220
targetPort: monitor-agent
protocol: TCP
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus-fluentd
namespace: fluentd
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus-k8s
subjects:
- kind: ServiceAccount
name: prometheus-fluentd
namespace: fluentd
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus-fluentd
namespace: fluentd
---
apiVersion: monitoring.coreos.com/v1alpha1
kind: Prometheus
metadata:
name: fluentd
namespace: fluentd
labels:
prometheus: fluentd
spec:
replicas: 1
serviceAccountName: prometheus-fluentd
serviceMonitorSelector:
matchLabels:
app: fluentd
resources:
requests:
memory: 250Mi
---
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment