Created
March 29, 2020 17:31
-
-
Save raj-prakarsh/8f63f1d2a4fe18260674538f158282c9 to your computer and use it in GitHub Desktop.
Added fluentd values for helm chart version 2.3.2 to push logs to elastic search and aws s3 bucket
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Default values for fluentd. | |
# This is a YAML-formatted file. | |
# Declare variables to be passed into your templates. | |
replicaCount: 3 | |
image: | |
repository: gcr.io/google-containers/fluentd-elasticsearch | |
tag: v2.4.0 | |
pullPolicy: IfNotPresent | |
# pullSecrets: | |
# - secret1 | |
# - secret2 | |
output: | |
host: <elasticsearch-client>.<namespace> | |
port: 9200 | |
scheme: http | |
sslVersion: TLSv1 | |
buffer_chunk_limit: 2M | |
buffer_queue_limit: 8 | |
env: {} | |
# Extra Environment Values - allows yaml definitions | |
extraEnvVars: | |
# - name: VALUE_FROM_SECRET | |
# valueFrom: | |
# secretKeyRef: | |
# name: secret_name | |
# key: secret_key | |
# extraVolumes: | |
# - name: es-certs | |
# secret: | |
# defaultMode: 420 | |
# secretName: es-certs | |
# extraVolumeMounts: | |
# - name: es-certs | |
# mountPath: /certs | |
# readOnly: true | |
plugins: | |
enabled: true | |
pluginsList: | |
- fluent-plugin-s3 | |
- fluent-plugin-rewrite-tag-filter | |
service: | |
annotations: {} | |
type: ClusterIP | |
# loadBalancerIP: | |
# type: NodePort | |
# nodePort: | |
# Used to create Service records | |
ports: | |
- name: "monitor-agent" | |
protocol: TCP | |
containerPort: 24220 | |
- name: "forward" | |
protocol: TCP | |
containerPort: 24224 | |
metrics: | |
enabled: false | |
service: | |
port: 24231 | |
serviceMonitor: | |
enabled: false | |
additionalLabels: {} | |
# namespace: monitoring | |
# interval: 30s | |
# scrapeTimeout: 10s | |
annotations: {} | |
# prometheus.io/scrape: "true" | |
# prometheus.io/port: "24231" | |
ingress: | |
enabled: false | |
annotations: | |
kubernetes.io/ingress.class: nginx | |
# kubernetes.io/tls-acme: "true" | |
# # Depending on which version of ingress controller you may need to configure properly - https://kubernetes.github.io/ingress-nginx/examples/rewrite/#rewrite-target | |
# nginx.ingress.kubernetes.io/rewrite-target: / | |
labels: [] | |
# If doing TCP or UDP ingress rule don't forget to update your Ingress Controller to accept TCP connections - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/ | |
hosts: | |
# - name: "http-input.local" | |
# protocol: TCP | |
# servicePort: 9880 | |
# path: / | |
tls: {} | |
# Secrets must be manually created in the namespace. | |
# - secretName: http-input-tls | |
# hosts: | |
# - http-input.local | |
configMaps: | |
general.conf: | | |
# Prevent fluentd from handling records containing its own logs. Otherwise | |
# it can lead to an infinite loop, when error in sending one message generates | |
# another message which also fails to be sent and so on. | |
<match fluentd.**> | |
@type null | |
</match> | |
# Used for health checking | |
<source> | |
@type http | |
port 9880 | |
bind 0.0.0.0 | |
</source> | |
# Emits internal metrics to every minute, and also exposes them on port | |
# 24220. Useful for determining if an output plugin is retryring/erroring, | |
# or determining the buffer queue length. | |
<source> | |
@type monitor_agent | |
bind 0.0.0.0 | |
port 24220 | |
tag fluentd.monitor.metrics | |
</source> | |
system.conf: |- | |
<system> | |
root_dir /tmp/fluentd-buffers/ | |
</system> | |
forward-input.conf: | | |
<source> | |
@type forward | |
port 24224 | |
bind 0.0.0.0 | |
</source> | |
output.conf: | | |
<filter kube.**> | |
@type record_transformer | |
enable_ruby | |
<record> | |
kubernetes_tag ${"%s" % [record["kubernetes"]["labels"]["app"] || record["kubernetes"]["labels"]["k8s-app"] || record["kubernetes"]["labels"]["name"] || "unspecified-app-label"]} | |
</record> | |
</filter> | |
<match kube.**> | |
@type rewrite_tag_filter | |
<rule> | |
key kubernetes_tag | |
pattern ^(.+)$ | |
tag $1 | |
</rule> | |
</match> | |
<match **> | |
@type s3 | |
s3_bucket <k8s-logs-bucket> | |
s3_region <ap-southeast-1> | |
s3_object_key_format "${tag}/%{time_slice}-events_%{index}.%{file_extension}" | |
time_slice_format %Y/%m/%d/%H | |
time_slice_wait 10m | |
path test-logs | |
# if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format, | |
# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument. | |
<buffer tag,time> | |
@type file | |
flush_mode interval | |
flush_interval 30s | |
path /var/log/fluent/s3 | |
timekey 300 # 1 hour partition | |
timekey_wait 1m | |
timekey_use_utc true # use utc | |
chunk_limit_size 100m | |
</buffer> | |
<format> | |
@type json | |
</format> | |
</match> | |
resources: | |
limits: | |
cpu: 500m | |
memory: 512Mi | |
requests: | |
cpu: 500m | |
memory: 512Mi | |
# We usually recommend not to specify default resources and to leave this as a conscious | |
# choice for the user. This also increases chances charts run on environments with little | |
# resources, such as Minikube. If you do want to specify resources, uncomment the following | |
# lines, adjust them as necessary, and remove the curly braces after 'resources:'. | |
# limits: | |
# cpu: 500m | |
# memory: 200Mi | |
# requests: | |
# cpu: 500m | |
# memory: 200Mi | |
rbac: | |
# Specifies whether RBAC resources should be created | |
create: true | |
serviceAccount: | |
# Specifies whether a ServiceAccount should be created | |
create: true | |
# The name of the ServiceAccount to use. | |
# If not set and create is true, a name is generated using the fullname template | |
name: | |
## Persist data to a persistent volume | |
persistence: | |
enabled: false | |
## If defined, storageClassName: <storageClass> | |
## If set to "-", storageClassName: "", which disables dynamic provisioning | |
## If undefined (the default) or set to null, no storageClassName spec is | |
## set, choosing the default provisioner. (gp2 on AWS, standard on | |
## GKE, AWS & OpenStack) | |
## | |
# storageClass: "-" | |
# annotations: {} | |
accessMode: ReadWriteOnce | |
size: 10Gi | |
nodeSelector: {} | |
tolerations: [] | |
affinity: {} | |
# Enable autoscaling using HorizontalPodAutoscaler | |
autoscaling: | |
enabled: false | |
minReplicas: 2 | |
maxReplicas: 5 | |
metrics: | |
- type: Resource | |
resource: | |
name: cpu | |
target: | |
type: Utilization | |
averageUtilization: 90 | |
- type: Resource | |
resource: | |
name: memory | |
target: | |
type: Utilization | |
averageUtilization: 80 | |
# Consider to set higher value when using in conjuction with autoscaling | |
# Full description about this field: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#pod-v1-core | |
terminationGracePeriodSeconds: 30 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment