Skip to content

Instantly share code, notes, and snippets.

@jcsorvasi

jcsorvasi/elasticsearch.yaml

Created April 19, 2021 12:52
Download ZIP

Revisions

  1. jcsorvasi created this gist Apr 19, 2021.
    597 changes: 597 additions & 0 deletions elasticsearch.yaml
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,597 @@
    apiVersion: elasticsearch.k8s.elastic.co/v1
    kind: Elasticsearch
    metadata:
    name: logging-prod
    namespace: logging-prod
    spec:
    version: 7.12.0
    http:
    tls:
    selfSignedCertificate:
    subjectAltNames:
    - dns: logging-prod-es-ingest # has to match any extra services created manually (not by the operator)
    service:
    spec:
    selector:
    common.k8s.elastic.co/type: elasticsearch
    elasticsearch.k8s.elastic.co/cluster-name: logging-prod
    elasticsearch.k8s.elastic.co/node-master: "false"
    elasticsearch.k8s.elastic.co/node-data: "false"
    elasticsearch.k8s.elastic.co/node-ingest: "false"
    elasticsearch.k8s.elastic.co/node-ml: "false"
    elasticsearch.k8s.elastic.co/node-transform: "false"
    nodeSets:
    - name: master-a
    count: 1
    config:
    node.roles: ["master"]
    xpack.ml.enabled: false
    node.attr.aws_availability_zone: eu-west-1a
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-master
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1a
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms10g -Xmx10g
    resources:
    requests:
    memory: 12Gi
    cpu: 4

    - name: master-b
    count: 1
    config:
    node.roles: ["master"]
    xpack.ml.enabled: false
    node.attr.aws_availability_zone: eu-west-1b
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-master
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1b
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms10g -Xmx10g
    resources:
    requests:
    memory: 12Gi
    cpu: 4

    - name: master-c
    count: 1
    config:
    node.roles: ["master"]
    xpack.ml.enabled: false
    node.attr.aws_availability_zone: eu-west-1c
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-master
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1c
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms10g -Xmx10g
    resources:
    requests:
    memory: 12Gi
    cpu: 4

    - name: data-hot-a
    count: 10
    config:
    node.roles: ["data"]
    node.attr.aws_availability_zone: eu-west-1a
    node.attr.tier: hot
    node.processors: 8
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-data
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1a
    containers:
    - name: elasticsearch
    env:
    - name: path.data
    value: "/mnt/data,/mnt/data2"
    - name: ES_JAVA_OPTS
    value: -Xms30g -Xmx30g
    resources:
    requests:
    memory: 32Gi
    cpu: 7
    volumeMounts:
    - name: elasticsearch-data
    mountPath: /mnt/data
    - name: elasticsearch-data2
    mountPath: /mnt/data2
    initContainers:
    - name: sysctl
    securityContext:
    privileged: true
    command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
    - command:
    [
    "sh",
    "-c",
    "chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
    ]
    name: chown-data-volumes
    securityContext:
    privileged: true
    volumeClaimTemplates:
    - metadata:
    name: elasticsearch-data
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage
    - metadata:
    name: elasticsearch-data2
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage

    - name: data-hot-b
    count: 10
    config:
    node.roles: ["data"]
    node.attr.aws_availability_zone: eu-west-1b
    node.attr.tier: hot
    node.processors: 8
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-data
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1b
    containers:
    - name: elasticsearch
    env:
    - name: path.data
    value: "/mnt/data,/mnt/data2"
    - name: ES_JAVA_OPTS
    value: -Xms30g -Xmx30g
    resources:
    requests:
    memory: 32Gi
    cpu: 7
    volumeMounts:
    - name: elasticsearch-data
    mountPath: /mnt/data
    - name: elasticsearch-data2
    mountPath: /mnt/data2
    initContainers:
    - name: sysctl
    securityContext:
    privileged: true
    command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
    - command:
    [
    "sh",
    "-c",
    "chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
    ]
    name: chown-data-volumes
    securityContext:
    privileged: true
    volumeClaimTemplates:
    - metadata:
    name: elasticsearch-data
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage
    - metadata:
    name: elasticsearch-data2
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage

    - name: data-hot-c
    count: 10
    config:
    node.roles: ["data"]
    node.attr.aws_availability_zone: eu-west-1c
    node.attr.tier: hot
    node.processors: 8
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-data
    tolerations:
    - effect: NoSchedule
    operator: Exists
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1c
    containers:
    - name: elasticsearch
    env:
    - name: path.data
    value: "/mnt/data,/mnt/data2"
    - name: ES_JAVA_OPTS
    value: -Xms30g -Xmx30g
    resources:
    requests:
    memory: 32Gi
    cpu: 7
    volumeMounts:
    - name: elasticsearch-data
    mountPath: /mnt/data
    - name: elasticsearch-data2
    mountPath: /mnt/data2
    initContainers:
    - name: sysctl
    securityContext:
    privileged: true
    command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
    - command:
    [
    "sh",
    "-c",
    "chown elasticsearch:elasticsearch /mnt/data && chown elasticsearch:elasticsearch /mnt/data2",
    ]
    name: chown-data-volumes
    securityContext:
    privileged: true
    volumeClaimTemplates:
    - metadata:
    name: elasticsearch-data
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage
    - metadata:
    name: elasticsearch-data2
    spec:
    accessModes:
    - ReadWriteOnce
    resources:
    requests:
    storage: 2061Gi
    storageClassName: local-storage
    - name: coordinating-a
    count: 1
    config:
    node.roles: []
    node.attr.aws_availability_zone: eu-west-1a
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1a
    - name: coordinating-b
    count: 1
    config:
    node.roles: []
    node.attr.aws_availability_zone: eu-west-1b
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1b
    - name: coordinating-c
    count: 1
    config:
    node.roles: []
    node.attr.aws_availability_zone: eu-west-1c
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1c
    - name: ingest-jmx-a
    count: 1
    config:
    node.roles: ["ingest", "transform"]
    node.attr.aws_availability_zone: eu-west-1a
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    logger.org.elasticsearch: DEBUG
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: "-Xms25g -Xmx25g -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9010"
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1a
    - name: ingest-a
    count: 1
    config:
    node.roles: ["ingest", "transform"]
    node.attr.aws_availability_zone: eu-west-1a
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1a
    - name: ingest-b
    count: 1
    config:
    node.roles: ["ingest", "transform"]
    node.attr.aws_availability_zone: eu-west-1b
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1b
    - name: ingest-c
    count: 1
    config:
    node.roles: ["ingest", "transform"]
    node.attr.aws_availability_zone: eu-west-1c
    cluster.routing.allocation.awareness.attributes: aws_availability_zone
    node.processors: 8
    xpack.ml.enabled: false
    podTemplate:
    spec:
    nodeSelector:
    foundation.meltwater.io/instance-class: es-client
    tolerations:
    - effect: NoSchedule
    operator: Exists
    containers:
    - name: elasticsearch
    env:
    - name: ES_JAVA_OPTS
    value: -Xms25g -Xmx25g
    resources:
    requests:
    memory: 28Gi
    cpu: 7
    affinity:
    nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    nodeSelectorTerms:
    - matchExpressions:
    - key: topology.kubernetes.io/zone
    operator: In
    values:
    - eu-west-1c
    ---
    apiVersion: v1
    kind: Service
    metadata:
    name: logging-prod-es-ingest
    labels:
    elasticsearch.k8s.elastic.co/cluster-name: logging-prod
    spec:
    ports:
    - name: https
    port: 9200
    protocol: TCP
    targetPort: 9200
    selector:
    common.k8s.elastic.co/type: elasticsearch
    elasticsearch.k8s.elastic.co/cluster-name: logging-prod
    elasticsearch.k8s.elastic.co/node-data: "false"
    elasticsearch.k8s.elastic.co/node-master: "false"
    elasticsearch.k8s.elastic.co/node-ingest: "true"