Skip to content

Instantly share code, notes, and snippets.

@jcsorvasi

jcsorvasi/elasticsearch.yaml

Created April 19, 2021 12:52
Download ZIP
Elasticsearch on ECK setup
Raw
elasticsearch.yaml
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: logging-prod
namespace: logging-prod
spec:
version: 7.12.0
http:
tls:
selfSignedCertificate:
subjectAltNames:
- dns: logging-prod-es-ingest # has to match any extra services created manually (not by the operator)
service:
spec:
selector:
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: logging-prod
elasticsearch.k8s.elastic.co/node-master: "false"
elasticsearch.k8s.elastic.co/node-data: "false"
elasticsearch.k8s.elastic.co/node-ingest: "false"
elasticsearch.k8s.elastic.co/node-ml: "false"
elasticsearch.k8s.elastic.co/node-transform: "false"
nodeSets:
- name: master-a
count: 1
config:
node.roles: ["master"]
xpack.ml.enabled: false
node.attr.aws_availability_zone: eu-west-1a
cluster.routing.allocation.awareness.attributes: aws_availability_zone
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-master
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1a
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms10g -Xmx10g
resources:
requests:
memory: 12Gi
cpu: 4
- name: master-b
count: 1
config:
node.roles: ["master"]
xpack.ml.enabled: false
node.attr.aws_availability_zone: eu-west-1b
cluster.routing.allocation.awareness.attributes: aws_availability_zone
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-master
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1b
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms10g -Xmx10g
resources:
requests:
memory: 12Gi
cpu: 4
- name: master-c
count: 1
config:
node.roles: ["master"]
xpack.ml.enabled: false
node.attr.aws_availability_zone: eu-west-1c
cluster.routing.allocation.awareness.attributes: aws_availability_zone
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-master
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1c
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms10g -Xmx10g
resources:
requests:
memory: 12Gi
cpu: 4
- name: data-hot-a
count: 10
config:
node.roles: ["data"]
node.attr.aws_availability_zone: eu-west-1a
node.attr.tier: hot
node.processors: 8
cluster.routing.allocation.awareness.attributes: aws_availability_zone
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-data
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1a
containers:
- name: elasticsearch
env:
- name: path.data
value: "/mnt/data,/mnt/data2"
- name: ES_JAVA_OPTS
value: -Xms30g -Xmx30g
resources:
requests:
memory: 32Gi
cpu: 7
volumeMounts:
- name: elasticsearch-data
mountPath: /mnt/data
- name: elasticsearch-data2
mountPath: /mnt/data2
initContainers:
- name: sysctl
securityContext:
privileged: true
command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
- command:
[
"sh",
"-c",
"chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
]
name: chown-data-volumes
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- metadata:
name: elasticsearch-data2
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- name: data-hot-b
count: 10
config:
node.roles: ["data"]
node.attr.aws_availability_zone: eu-west-1b
node.attr.tier: hot
node.processors: 8
cluster.routing.allocation.awareness.attributes: aws_availability_zone
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-data
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1b
containers:
- name: elasticsearch
env:
- name: path.data
value: "/mnt/data,/mnt/data2"
- name: ES_JAVA_OPTS
value: -Xms30g -Xmx30g
resources:
requests:
memory: 32Gi
cpu: 7
volumeMounts:
- name: elasticsearch-data
mountPath: /mnt/data
- name: elasticsearch-data2
mountPath: /mnt/data2
initContainers:
- name: sysctl
securityContext:
privileged: true
command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
- command:
[
"sh",
"-c",
"chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
]
name: chown-data-volumes
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- metadata:
name: elasticsearch-data2
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- name: data-hot-c
count: 10
config:
node.roles: ["data"]
node.attr.aws_availability_zone: eu-west-1c
node.attr.tier: hot
node.processors: 8
cluster.routing.allocation.awareness.attributes: aws_availability_zone
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-data
tolerations:
- effect: NoSchedule
operator: Exists
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1c
containers:
- name: elasticsearch
env:
- name: path.data
value: "/mnt/data,/mnt/data2"
- name: ES_JAVA_OPTS
value: -Xms30g -Xmx30g
resources:
requests:
memory: 32Gi
cpu: 7
volumeMounts:
- name: elasticsearch-data
mountPath: /mnt/data
- name: elasticsearch-data2
mountPath: /mnt/data2
initContainers:
- name: sysctl
securityContext:
privileged: true
command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
- command:
[
"sh",
"-c",
"chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
]
name: chown-data-volumes
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- metadata:
name: elasticsearch-data2
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2061Gi
storageClassName: local-storage
- name: coordinating-a
count: 1
config:
node.roles: []
node.attr.aws_availability_zone: eu-west-1a
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1a
- name: coordinating-b
count: 1
config:
node.roles: []
node.attr.aws_availability_zone: eu-west-1b
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1b
- name: coordinating-c
count: 1
config:
node.roles: []
node.attr.aws_availability_zone: eu-west-1c
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1c
- name: ingest-jmx-a
count: 1
config:
node.roles: ["ingest", "transform"]
node.attr.aws_availability_zone: eu-west-1a
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
logger.org.elasticsearch: DEBUG
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: "-Xms25g -Xmx25g -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9010"
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1a
- name: ingest-a
count: 1
config:
node.roles: ["ingest", "transform"]
node.attr.aws_availability_zone: eu-west-1a
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1a
- name: ingest-b
count: 1
config:
node.roles: ["ingest", "transform"]
node.attr.aws_availability_zone: eu-west-1b
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1b
- name: ingest-c
count: 1
config:
node.roles: ["ingest", "transform"]
node.attr.aws_availability_zone: eu-west-1c
cluster.routing.allocation.awareness.attributes: aws_availability_zone
node.processors: 8
xpack.ml.enabled: false
podTemplate:
spec:
nodeSelector:
foundation.meltwater.io/instance-class: es-client
tolerations:
- effect: NoSchedule
operator: Exists
containers:
- name: elasticsearch
env:
- name: ES_JAVA_OPTS
value: -Xms25g -Xmx25g
resources:
requests:
memory: 28Gi
cpu: 7
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- eu-west-1c
---
apiVersion: v1
kind: Service
metadata:
name: logging-prod-es-ingest
labels:
elasticsearch.k8s.elastic.co/cluster-name: logging-prod
spec:
ports:
- name: https
port: 9200
protocol: TCP
targetPort: 9200
selector:
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: logging-prod
elasticsearch.k8s.elastic.co/node-data: "false"
elasticsearch.k8s.elastic.co/node-master: "false"
elasticsearch.k8s.elastic.co/node-ingest: "true"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment