Skip to content

Instantly share code, notes, and snippets.

@mweibel
Created July 5, 2021 08:18
Show Gist options
  • Star 5 You must be signed in to star a gist
  • Fork 3 You must be signed in to fork a gist
  • Save mweibel/a44dd5de1bd7040b2d1aeb6889c105ac to your computer and use it in GitHub Desktop.
Save mweibel/a44dd5de1bd7040b2d1aeb6889c105ac to your computer and use it in GitHub Desktop.
banzaicloud/logging-operator working ElasticSearch example with index lifecycle management
apiVersion: v1
kind: Secret
metadata:
name: index-template
stringData:
template: '{"index_patterns":["logstash-*"],"template":{"aliases":{"logstash":{}},"mappings":{"dynamic":"true","dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"dynamic_templates":[]},"settings":{"index":{"lifecycle":{"name":"logstash_policy","rollover_alias":"logstash"},"number_of_shards":"1","number_of_replicas":"0"}}}}'
# ClusterFlows to deploy
clusterFlows:
- name: all
spec:
filters:
- record_modifier: # if you e.g. have multiple clusters
records:
- cluster: "CLUSTER_NAME"
# replaces dots in labels and annotations with dashes to avoid mapping issues (app=foo (text) vs. app.kubernetes.io/name=foo (object))
# fixes error: existing mapping for [kubernetes.labels.app] must be of type object but found [text]
- dedot:
de_dot_separator: "-"
de_dot_nested: true
globalOutputRefs:
- elasticsearch
# ClusterOutputs to deploy
clusterOutputs:
- name: elasticsearch
spec:
elasticsearch:
host: ES_HOST
port: EST_PORT
user: ES_USER
password:
valueFrom:
secretKeyRef:
name: es-password
key: password
scheme: https
ssl_verify: true
logstash_format: true
include_timestamp: true
reconnect_on_error: true
reload_on_failure: true
buffer:
flush_mode: interval
flush_interval: 60s
flush_at_shutdown: true
flush_thread_count: 4
enable_ilm: true
ilm_policy_id: logstash_policy
ilm_policy: '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"readonly":{},"rollover":{"max_size":"8gb","max_primary_shard_size":"4gb","max_age":"7d"}}},"cold":{"min_age":"7d","actions":{"freeze":{},"readonly":{},"searchable_snapshot":{"snapshot_repository":"found-snapshots","force_merge_index":true},"set_priority":{"priority":0}}}}}}'
# ilm_policy_overwrite: false # # when updating the existing policy, this needs to be set to true and then redeployed with false to avoid versioning of this policy every 60secs
template_name: "logstash_template"
template_file:
mountFrom:
secretKeyRef:
name: index-template
key: template
customize_template: '{"settings.index.number_of_replicas": "0"}'
# enables logging of bad request reasons within the fluentd log file (in the pod /fluentd/log/out)
log_es_400_reason: true
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment