Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
very very basic kubernetes fluentd setup
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: fluentd
namespace: kube-system
labels:
k8s-app: fluentd-logging
version: v1
kubernetes.io/cluster-service: "true"
spec:
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: fluentd-logging
version: v1
kubernetes.io/cluster-service: "true"
spec:
serviceAccountName: fluentd
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: fluentd
image: fluent/fluentd-kubernetes-daemonset:v0.12-debian-cloudwatch
command:
- sh
args:
- -c
- "locale-gen en_US.UTF-8; apt-get update; apt-get install -y make gcc g++ libc-dev ruby-dev libffi-dev; gem install fluent-plugin-systemd; gem install systemd-journal; gem install fluent-plugin-rewrite-tag-filter; gem install fluent-plugin-record-modifier; fluentd -c /fluentd/etc/fluent.conf -p /fluentd/plugins"
env:
- name: AWS_ACCESS_KEY_ID
value: hihi
- name: AWS_SECRET_ACCESS_KEY
value: hihi
- name: AWS_DEFAULT_REGION
value: "eu-west-2"
- name: FLUENT_HOSTNAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: LOG_GROUP_NAME
value: "k8s.home.internal.randomvariable.co.uk"
- name: LANG
value: en_US.UTF-8
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: journal
mountPath: /var/log/journal
- name: config-volume
mountPath: /fluentd/etc
terminationGracePeriodSeconds: 5
volumes:
- name: varlog
hostPath:
path: /var/log
- name: journal
hostPath:
path: /var/log/journal
- name: config-volume
configMap:
name: fluentd-config
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: kube-system
labels:
k8s-app: fluentd-logging
version: v1
kubernetes.io/cluster-service: "true"
data:
fluent.conf: |-
#{Encoding.default_internal = Encoding::UTF_8}
#{Encoding.default_external = Encoding::UTF_8}
@include systemd.conf
@include kubernetes.conf
<filter forward.**>
@type record_transformer
<record>
log_tag ${tag_suffix[1]}
</record>
</filter>
<filter fluent.**>
@type record_transformer
<record>
log_tag fluent
</record>
</filter>
<filter **>
@type record_modifier
char_encoding utf-8
</filter>
<match forward.**>
@type rewrite_tag_filter
<rule>
key log_tag
pattern (.+)
tag "logs.$1.#{ENV["FLUENT_HOSTNAME"]}"
</rule>
</match>
<match fluent.**>
@type rewrite_tag_filter
<rule>
key log_tag
pattern fluent
tag "logs.fluent.#{ENV["FLUENT_HOSTNAME"]}"
</rule>
</match>
<match logs.**>
@type cloudwatch_logs
log_group_name "#{ENV['LOG_GROUP_NAME']}"
auto_create_stream true
use_tag_as_stream true
remove_log_stream_name_key log_tag
</match>
<match fluent.**>
@type cloudwatch_logs
log_group_name "#{ENV['LOG_GROUP_NAME']}"
auto_create_stream true
log_stream_name "logs.fluent.#{ENV["FLUENT_HOSTNAME"]}"
</match>
kubernetes.conf: |-
<source>
@type tail
path /var/log/kubernetes/audit/*.log
pos_file /var/log/fluentd-kubernetes-audit.log.pos
time_format %Y-%m-%dT%H:%M:%S.%NZ
tag forward.kubernetes.audit
format json
read_from_head true
</source>
systemd.conf: |-
<source>
@id systemd
@type systemd
<storage>
@type local
path journald.pos
persistent true
</storage>
read_from_head true
tag forward.journal
</source>
<source>
@type systemd
tag forward.kubernetes.containers
path /var/log/journal
filters [{ "_COMM": "dockerd-current" }, {"_TRANSPORT": "journal"}]
read_from_head true
<storage>
@type local
persistent true
path kubernetes.pos
</storage>
</source>
<filter forward.kubernetes.containers>
@type kubernetes_metadata
use_journal true
</filter>
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
namespace: kube-system
name: fluentd-role
rules:
- apiGroups: [""] # "" indicates the core API group
resources: ["pods", "namespaces"]
verbs: ["get", "watch", "list"]
---
# This role binding allows "system:serviceaccount:logs:default" to read pods in the "default" namespace.
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: fluentd-service-account
namespace: kube-system
subjects:
- kind: ServiceAccount
name: fluentd
namespace: kube-system
roleRef:
kind: ClusterRole
name: fluentd-role
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: kube-system
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.