Created
August 30, 2019 02:52
-
-
Save silasb/9232942b2f4995bd7541b41a392f22ae to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: fluent-bit-config | |
namespace: logging | |
labels: | |
k8s-app: fluent-bit | |
data: | |
# Configuration files: server, input, filters and output | |
# ====================================================== | |
fluent-bit.conf: | | |
[SERVICE] | |
Flush 1 | |
Log_Level info | |
Daemon off | |
Parsers_File parsers.conf | |
HTTP_Server On | |
HTTP_Listen 0.0.0.0 | |
HTTP_Port 2020 | |
@INCLUDE input-kubernetes.conf | |
@INCLUDE filter-kubernetes.conf | |
@INCLUDE output-elasticsearch.conf | |
input-kubernetes.conf: | | |
[INPUT] | |
Name tail | |
Tag kube.* | |
Path /var/log/containers/*.log | |
Parser docker | |
DB /var/log/flb_kube.db | |
Mem_Buf_Limit 5MB | |
Skip_Long_Lines On | |
Refresh_Interval 10 | |
filter-kubernetes.conf: | | |
[FILTER] | |
Name kubernetes | |
Match kube.* | |
Kube_URL https://kubernetes.default.svc:443 | |
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token | |
Kube_Tag_Prefix kube.var.log.containers. | |
Merge_Log On | |
Merge_Log_Key log_processed | |
K8S-Logging.Parser On | |
K8S-Logging.Exclude Off | |
[FILTER] | |
Name nest | |
Match kube.* | |
Operation lift | |
Nested_under kubernetes | |
[FILTER] | |
Name grep | |
Match kube.* | |
Exclude namespace_name kube-system | |
Exclude namespace_name openebs | |
Exclude namespace_name metallb-system | |
Exclude namespace_name logging | |
output-elasticsearch.conf: | | |
[OUTPUT] | |
Name stackdriver | |
Match * | |
google_service_credentials /etc/config/gcp.json | |
resource global | |
tls On | |
tls.verify Off | |
# tls.debug 4 | |
# IPv6 On | |
parsers.conf: | | |
[PARSER] | |
Name apache | |
Format regex | |
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name apache2 | |
Format regex | |
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name apache_error | |
Format regex | |
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$ | |
[PARSER] | |
Name nginx | |
Format regex | |
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name json | |
Format json | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name docker | |
Format json | |
Time_Key time | |
Time_Format %Y-%m-%dT%H:%M:%S.%L | |
Time_Keep On | |
[PARSER] | |
Name syslog | |
Format regex | |
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$ | |
Time_Key time | |
Time_Format %b %d %H:%M:%S |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: extensions/v1beta1 | |
kind: DaemonSet | |
metadata: | |
name: fluent-bit | |
namespace: logging | |
labels: | |
k8s-app: fluent-bit-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
spec: | |
template: | |
metadata: | |
labels: | |
k8s-app: fluent-bit-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
annotations: | |
prometheus.io/scrape: "true" | |
prometheus.io/port: "2020" | |
prometheus.io/path: /api/v1/metrics/prometheus | |
spec: | |
initContainers: | |
- name: init-test-container | |
image: alpine:3.10 | |
command: [ "/bin/sh", "-c", "base64 -d /etc/config/gcp.json.base64 > /pod-data/gcp.json" ] | |
volumeMounts: | |
- name: gcp-auth | |
mountPath: /etc/config | |
- name: shared-data | |
mountPath: /pod-data | |
containers: | |
- name: fluent-bit | |
image: fluent/fluent-bit:1.2.1 | |
imagePullPolicy: Always | |
ports: | |
- containerPort: 2020 | |
env: | |
volumeMounts: | |
- name: varlog | |
mountPath: /var/log | |
- name: varlibdockercontainers | |
mountPath: /var/lib/docker/containers | |
readOnly: true | |
- name: fluent-bit-config | |
mountPath: /fluent-bit/etc/ | |
- name: shared-data | |
mountPath: /etc/config | |
terminationGracePeriodSeconds: 10 | |
volumes: | |
- name: shared-data | |
emptyDir: {} | |
- name: varlog | |
hostPath: | |
path: /var/log | |
- name: varlibdockercontainers | |
hostPath: | |
path: /var/lib/docker/containers | |
- name: fluent-bit-config | |
configMap: | |
name: fluent-bit-config | |
- name: gcp-auth | |
configMap: | |
name: gcp-auth | |
serviceAccountName: fluent-bit | |
tolerations: | |
- key: node-role.kubernetes.io/master | |
operator: Exists | |
effect: NoSchedule | |
- operator: "Exists" | |
effect: "NoExecute" | |
- operator: "Exists" | |
effect: "NoSchedule" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
kubectl create secret generic gcp-auth --dry-run --from-literal=gcp.json.base64=$(base64 -w0 gcp.json) -o json >gcp-auth-secret.json | |
# do a quick verification to make sure the secret decodes fine | |
# this should be the same as your gcp.json file | |
cat gcp-auth-secret.json | jq '.data | .["gcp.json.base64"]' -r | base64 -d | base64 -d | |
kubeseal <gcp-auth-secret.json >gcp-auth-sealedsecret.json | |
kubectl apply -f gcp-auth-sealedsecret.json | |
# do another test to verify the gcp-auth is the same as gcp.json | |
kubectl get secrets gcp-auth -ojson | jq '.data | .["gcp.json.base64"]' -r| base64 -d | base64 -d |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment