Last active
December 11, 2023 08:26
-
-
Save arn-ob/4475bfa39f55685450d7a7b345fbd74f to your computer and use it in GitHub Desktop.
ElasticSearch snapshot and restore Kubernetes Yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
kind: Namespace | |
metadata: | |
name: elasticsearch | |
--- | |
apiVersion: elasticsearch.k8s.elastic.co/v1 | |
kind: Elasticsearch | |
metadata: | |
name: quickstart | |
namespace: elasticsearch | |
spec: | |
version: 8.10.2 | |
nodeSets: | |
- name: default | |
count: 1 | |
podTemplate: | |
spec: | |
initContainers: | |
- name: install-plugins | |
command: | |
- sh | |
- -c | |
- | | |
bin/elasticsearch-plugin install --batch repository-s3 | |
- name: add-aws-keys | |
env: | |
- name: AWS_ACCESS_KEY_ID | |
value: <AWS_ACCESS_KEY_ID> | |
- name: AWS_SECRET_ACCESS_KEY | |
value: <AWS_SECRET_ACCESS_KEY> | |
command: | |
- sh | |
- -c | |
- | | |
echo $AWS_ACCESS_KEY_ID | bin/elasticsearch-keystore add --stdin --force s3.client.default.access_key | |
echo $AWS_SECRET_ACCESS_KEY | bin/elasticsearch-keystore add --stdin --force s3.client.default.secret_key | |
volumeClaimTemplates: | |
- metadata: | |
name: elasticsearch-data | |
spec: | |
accessModes: | |
- ReadWriteOnce | |
resources: | |
requests: | |
storage: 100Gi | |
config: | |
node.store.allow_mmap: false | |
--- | |
apiVersion: kibana.k8s.elastic.co/v1 | |
kind: Kibana | |
metadata: | |
name: quickstart | |
namespace: elasticsearch | |
spec: | |
version: 8.10.2 | |
count: 1 | |
elasticsearchRef: | |
name: quickstart | |
http: | |
tls: | |
selfSignedCertificate: | |
disabled: true | |
--- | |
apiVersion: networking.k8s.io/v1 | |
kind: Ingress | |
metadata: | |
name: elastic-search-api-ingress | |
namespace: elasticsearch | |
annotations: | |
kubernetes.io/ingress.class: nginx | |
kubernetes.io/tls-acme: "true" | |
nginx.ingress.kubernetes.io/proxy-ssl-verify: "false" | |
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" | |
spec: | |
rules: | |
- host: elasticsearch-api.example.com | |
http: | |
paths: | |
- path: / | |
pathType: Prefix | |
backend: | |
service: | |
name: quickstart-es-http | |
port: | |
number: 9200 | |
--- | |
apiVersion: networking.k8s.io/v1 | |
kind: Ingress | |
metadata: | |
name: elastic-search-ingress | |
namespace: elasticsearch | |
annotations: | |
kubernetes.io/ingress.class: nginx | |
spec: | |
rules: | |
- host: elasticsearch.example.com | |
http: | |
paths: | |
- path: / | |
pathType: Prefix | |
backend: | |
service: | |
name: quickstart-kb-http | |
port: | |
number: 5601 | |
--- | |
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: fluent-bit-config | |
namespace: logging | |
labels: | |
k8s-app: fluent-bit | |
data: | |
fluent-bit.conf: | | |
[SERVICE] | |
Flush 1 | |
Log_Level info | |
Daemon off | |
Parsers_File parsers.conf | |
HTTP_Server On | |
HTTP_Listen 0.0.0.0 | |
HTTP_Port 2020 | |
@INCLUDE input-kubernetes.conf | |
@INCLUDE filter-kubernetes.conf | |
@INCLUDE output-elasticsearch.conf | |
input-kubernetes.conf: | | |
[INPUT] | |
Name tail | |
Tag kube.* | |
Path /var/log/containers/*.log | |
Exclude_Path /var/log/containers/*_kube-system_*.log,/var/log/containers/*_kube-public_*.log,/var/log/containers/*_kube-node-lease_*.log,/var/log/containers/*_ingress-nginx_*.log,/var/log/containers/*_prometheus_*.log,/var/log/containers/*_argocd_*.log,/var/log/containers/*_cert-manager_*.log,/var/log/containers/*_chatwoot_*.log,/var/log/containers/*_elastic-system_*.log,/var/log/containers/*_elasticsearch_*.log,/var/log/containers/*_logging_*.log | |
Parser docker | |
DB /var/log/flb_kube.db | |
Mem_Buf_Limit 5MB | |
Skip_Long_Lines On | |
Refresh_Interval 10 | |
filter-kubernetes.conf: | | |
[FILTER] | |
Name kubernetes | |
Match kube.* | |
Kube_URL https://kubernetes.default.svc:443 | |
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | |
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token | |
Kube_Tag_Prefix kube.var.log.containers. | |
Merge_Log On | |
Merge_Log_Key log_processed | |
K8S-Logging.Parser On | |
K8S-Logging.Exclude Off | |
output-elasticsearch.conf: | | |
[OUTPUT] | |
Name es | |
Match * | |
Host quickstart-es-http.elasticsearch | |
Port 9200 | |
HTTP_User elastic | |
HTTP_Passwd <Password> | |
tls On | |
tls.verify Off | |
Logstash_Format On | |
Replace_Dots On | |
Suppress_Type_Name On | |
Retry_Limit False | |
parsers.conf: | | |
[PARSER] | |
Name apache | |
Format regex | |
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name apache2 | |
Format regex | |
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name apache_error | |
Format regex | |
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$ | |
[PARSER] | |
Name nginx | |
Format regex | |
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name json | |
Format json | |
Time_Key time | |
Time_Format %d/%b/%Y:%H:%M:%S %z | |
[PARSER] | |
Name docker | |
Format json | |
Time_Key time | |
Time_Format %Y-%m-%dT%H:%M:%S.%L | |
Time_Keep On | |
[PARSER] | |
Name cri | |
Format regex | |
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$ | |
Time_Key time | |
Time_Format %Y-%m-%dT%H:%M:%S.%L%z | |
[PARSER] | |
Name syslog | |
Format regex | |
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$ | |
Time_Key time | |
Time_Format %b %d %H:%M:%S | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1 | |
kind: ClusterRoleBinding | |
metadata: | |
name: fluent-bit-read | |
roleRef: | |
apiGroup: rbac.authorization.k8s.io | |
kind: ClusterRole | |
name: fluent-bit-read | |
subjects: | |
- kind: ServiceAccount | |
name: fluent-bit | |
namespace: logging | |
--- | |
apiVersion: rbac.authorization.k8s.io/v1 | |
kind: ClusterRole | |
metadata: | |
name: fluent-bit-read | |
rules: | |
- apiGroups: [""] | |
resources: | |
- namespaces | |
- pods | |
verbs: ["get", "list", "watch"] | |
--- | |
apiVersion: apps/v1 | |
kind: DaemonSet | |
metadata: | |
name: fluent-bit | |
namespace: logging | |
labels: | |
k8s-app: fluent-bit-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
spec: | |
selector: | |
matchLabels: | |
k8s-app: fluent-bit-logging | |
template: | |
metadata: | |
labels: | |
k8s-app: fluent-bit-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
annotations: | |
prometheus.io/scrape: "true" | |
prometheus.io/port: "2020" | |
prometheus.io/path: /api/v1/metrics/prometheus | |
spec: | |
containers: | |
- name: fluent-bit | |
image: fluent/fluent-bit:latest | |
imagePullPolicy: Always | |
ports: | |
- containerPort: 2020 | |
volumeMounts: | |
- name: varlog | |
mountPath: /var/log | |
- name: varlibdockercontainers | |
mountPath: /var/lib/docker/containers | |
readOnly: true | |
- name: fluent-bit-config | |
mountPath: /fluent-bit/etc/ | |
terminationGracePeriodSeconds: 10 | |
volumes: | |
- name: varlog | |
hostPath: | |
path: /var/log | |
- name: varlibdockercontainers | |
hostPath: | |
path: /var/lib/docker/containers | |
- name: fluent-bit-config | |
configMap: | |
name: fluent-bit-config | |
serviceAccountName: fluent-bit | |
tolerations: | |
- key: node-role.kubernetes.io/master | |
operator: Exists | |
effect: NoSchedule | |
- operator: "Exists" | |
effect: "NoExecute" | |
- operator: "Exists" | |
effect: "NoSchedule" | |
--- | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: fluent-bit | |
namespace: logging |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment