Skip to content

Instantly share code, notes, and snippets.

@tkuther
Created July 12, 2018 12:33
Show Gist options
  • Save tkuther/b432827283f360293a87b5be49594f91 to your computer and use it in GitHub Desktop.
Save tkuther/b432827283f360293a87b5be49594f91 to your computer and use it in GitHub Desktop.
Filebeat kubernetes config with nginx module for ingress-nginx
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
spec:
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.3.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: inputs
mountPath: /usr/share/filebeat/inputs.d
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: config
mountPath: /usr/share/filebeat/module/nginx/access/ingest/default.json
readOnly: true
subPath: default.json
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: inputs
configMap:
defaultMode: 0600
name: filebeat-inputs
- name: data
hostPath:
path: /var/lib/filebeat
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-inputs
namespace: kube-system
labels:
k8s-app: filebeat
data:
kubernetes.yml: |-
- type: docker
containers.ids:
- "*"
processors:
- add_kubernetes_metadata:
in_cluster: true
- drop_event:
when:
equals:
kubernetes.labels.app: nginx-ingress
exclude_lines: ['kube-probe']
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.config:
inputs:
# Mounted `filebeat-inputs` configmap:
path: ${path.config}/inputs.d/*.yml
# Reload inputs configs as they change:
reload.enabled: false
modules:
path: ${path.config}/modules.d/*.yml
# Reload module configs as they change:
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition:
equals:
kubernetes.labels.app: nginx-ingress
config:
- module: nginx
access:
input:
type: docker
containers.ids:
- "${data.kubernetes.container.id}"
processors:
- add_cloud_metadata:
fields:
logtype: kubernetes
kubernetes.cluster.name: aks-ae-01
fields_under_root: true
output.elasticsearch:
hosts: ["es-master.elasticsearch.svc.cluster.local:9200"]
default.json: |-
{
"description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"\"?%{IP_LIST:nginx.access.remote_ip_list} - \\[%{IP_LIST:nginx.access.x_forwarded_for}\\] - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} (?:%{NUMBER:nginx.access.body_sent.bytes}|-) (?:\"(?:%{DATA:nginx.access.referrer}|-)\") \"%{DATA:nginx.access.agent}\" %{NUMBER:nginx.access.request_length} %{NUMBER:nginx.access.response_time} %{DATA:nginx.access.upstream.proxy} %{NUMBER:nginx.access.upstream.bytes_sent} %{NUMBER:nginx.access.upstream.response_time} %{NUMBER:nginx.access.upstream.response}",
"\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
],
"pattern_definitions": {
"IP_LIST": "%{IP}(\"?,?\\s*%{IP})*"
},
"ignore_missing": true
}
}, {
"grok": {
"field": "nginx.access.info",
"patterns": [
"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}",
""
],
"ignore_missing": true
}
}, {
"remove": {
"field": "nginx.access.info"
}
}, {
"split": {
"field": "nginx.access.remote_ip_list",
"separator": "\"?,?\\s+"
}
}, {
"script": {
"lang": "painless",
"inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken());if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(item)) { ctx.nginx.access.remote_ip = item; found = true; break; } } if (!found) { ctx.nginx.access.remote_ip = ctx.nginx.access.remote_ip_list[0]; }"
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
}, {
"date": {
"field": "nginx.access.time",
"target_field": "@timestamp",
"formats": ["dd/MMM/YYYY:H:m:s Z"]
}
}, {
"remove": {
"field": "nginx.access.time"
}
}, {
"user_agent": {
"field": "nginx.access.agent",
"target_field": "nginx.access.user_agent"
}
}, {
"remove": {
"field": "nginx.access.agent"
}
}, {
"geoip": {
"field": "nginx.access.remote_ip",
"target_field": "nginx.access.geoip"
}
}],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}
@EvgeniGordeev
Copy link

It doesn't work since 7.11 any more.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment