Configuring Elasticsearch 7 cluster on GKE

commands-gke.txt

#
# variables
#
export PROJECT=$(gcloud config get-value project)
export REGISTRY_HOST=asia.gcr.io
export CONTAINER_IMAGE=my-elasticsearch-7
export IMAGE_VERSION=0.1.0
export GKE_CLUSTER=elasticsearch

#
# authentication
#
gcloud init
gcloud auth configure-docker

#
# build and push docker image
#
docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-${IMAGE_VERSION} --no-cache .
docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION}
docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION}
gcloud container images describe ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION}

#
# connect to GKE cluster
# note: before executing, you have to create a GKE cluster named 'elasticsearch'.
#
gcloud container clusters get-credentials ${GKE_CLUSTER} --region asia-northeast1 --project ${PROJECT}
kubectl get nodes

#
# apply manifests
#
kubectl apply -f configmap-common.yaml 
kubectl apply -f configmap-data.yaml 
kubectl apply -f configmap-master.yaml 
kubectl apply -f service-data.yaml 
kubectl apply -f service-master.yaml 
kubectl apply -f storageclass-ssd.yaml
kubectl apply -f statefulset-master.yaml
kubectl apply -f statefulset-data.yaml

# verify if the cluster is properly created
kubectl get nodes
kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty


#####################################################
# commands for veryfing StatefulSet auto recovering #
#####################################################
# index test data
kubectl exec -it elasticsearch-data-0 -- curl -XPUT localhost:9200/testidx -HContent-type:application/json --data '{"settings": {"index": {"number_of_shards": 1, "number_of_replicas": 2}}}'
for i in {1..100}; do kubectl exec elasticsearch-data-0 -- curl -s -XPOST localhost:9200/testidx/_doc -HContent-type:application/json --data '{"data":"test"}'; done

kubectl exec -it elasticsearch-data-0 -- curl "localhost:9200/testidx/_search?size=0&pretty"
kubectl exec -it elasticsearch-data-0 -- curl "localhost:9200/_nodes/stats/indices/docs?pretty"

# delete a data node
kubectl delete pod elasticsearch-data-1

# verify auto recoverying
watch kubectl get pods
kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty

configmap-common.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-common-config
data:
  bootstrap.memory_lock: "false" # can't lock memory on GKE. instead, set "vm.swappiness=1" when initializing containers. (see the StatefulSet manifests.)
  http.port: "9200"
  cluster.initial_master_nodes: "elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2"
  cluster.name: "my-es-cluster"
  discovery.seed_hosts: "elasticsearch-master"
  ES_JAVA_OPTS: -Xms1g -Xmx1g  # for testing 
  xpack.license.self_generated.type: "basic"
  xpack.ml.enabled: "false"
  xpack.monitoring.collection.enabled: "false"
  xpack.monitoring.enabled: "false"
  xpack.security.enabled: "false"
  xpack.sql.enabled: "false"
  xpack.watcher.enabled: "false"

configmap-data.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-data-config
data:
  node.data: "true"
  node.ingest: "false"
  node.master: "false"
  node.ml: "false"

configmap-master.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch-master-config
data:
  node.data: "false"
  node.ingest: "false"
  node.master: "true"
  node.ml: "false"

Dockerfile-0.1.0

FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0

ENV PATH /usr/share/elasticsearch/bin:$PATH

# switch user to elasticsearch
USER elasticsearch

# install plugins
RUN elasticsearch-plugin install analysis-kuromoji

Dockerfile-0.2.0

FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0

ENV PATH /usr/share/elasticsearch/bin:$PATH

# switch user to elasticsearch
USER elasticsearch

# install plugins
RUN elasticsearch-plugin install analysis-kuromoji
RUN elasticsearch-plugin install analysis-icu

service-data.yaml

apiVersion: v1
kind: Service
metadata:
  name: elasticsearch-data
spec:
  selector:
    app: elasticsearch-data
  type: ClusterIP
  ports:
  - protocol: TCP
    name: rest
    port: 9200
    targetPort: 9200
  - protocol: TCP
    name: nodes
    port: 9300
    targetPort: 9300

service-master.yaml

apiVersion: v1
kind: Service
metadata:
  name: elasticsearch-master
spec:
  selector:
    app: elasticsearch-master
  type: ClusterIP
  ports:
  - protocol: TCP
    name: rest
    port: 9200
    targetPort: 9200
  - protocol: TCP
    name: nodes
    port: 9300
    targetPort: 9300

statefulset-data.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch-data
spec:
  selector:
    matchLabels:
      app: elasticsearch-data
  serviceName: elasticsearch-data
  replicas: 3
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: elasticsearch-data
    spec:
      initContainers:
      - name: increase-the-ulimit
        image: busybox
        command: ["sh", "-c", "ulimit -n 65536"]
        securityContext:
          privileged: true
      - name: configure-swappiness
        image: busybox
        command: ["sh", "-c", "sysctl -w vm.swappiness=1"]
        securityContext:
          privileged: true
      - name: max-map-count
        image: busybox
        command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
        securityContext:
          privileged: true
      - name: chown-mountpoint
        image: busybox
        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: es-data
          mountPath: /usr/share/elasticsearch/data
      terminationGracePeriodSeconds: 300
      containers:
      - name: elasticsearch
        image: asia.gcr.io/my-project-may31/my-elasticsearch-7:0.1.0  # FIXME
        resources:
          requests:
            cpu: "400m"  # for testing
        ports:
        - containerPort: 9200
          name: rest
        #readinessProbe: # Note: disable the readiness probe only when cluster bootstrapping
        #  exec:
        #    command:
        #    - "sh"
        #    - "-c"
        #    - "curl -s http://localhost:9200/ | grep 'cluster_uuid' | grep -v '_na_'"  # check whether cluster_uuid is set
        #  periodSeconds: 20
        livenessProbe:
          httpGet:
            port: 9200
            path: /
          initialDelaySeconds: 300
          periodSeconds: 20
        volumeMounts:
        - name: es-data
          mountPath: /usr/share/elasticsearch/data
        envFrom:
        - configMapRef:
            name: elasticsearch-common-config
        - configMapRef:
            name: elasticsearch-data-config
  volumeClaimTemplates:
  - metadata:
      name: es-data
    spec:
      storageClassName: ssd
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 100Gi

statefulset-master.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch-master
spec:
  selector:
    matchLabels:
      app: elasticsearch-master
  serviceName: elasticsearch-master
  replicas: 3
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: elasticsearch-master
    spec:
      initContainers:
      - name: increase-the-ulimit
        image: busybox
        command: ["sh", "-c", "ulimit -n 65536"]
        securityContext:
          privileged: true
      - name: configure-swappiness
        image: busybox
        command: ["sh", "-c", "sysctl -w vm.swappiness=1"]
        securityContext:
          privileged: true
      - name: max-map-count
        image: busybox
        command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"]
        securityContext:
          privileged: true
      - name: chown-mountpoint
        image: busybox
        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: es-data
          mountPath: /usr/share/elasticsearch/data
      terminationGracePeriodSeconds: 300
      containers:
      - name: elasticsearch
        image: asia.gcr.io/my-project-may31/my-elasticsearch-7:0.1.0  # FIXME
        resources:
          requests:
            cpu: "400m" # for testing
        ports:
        - containerPort: 9200
          name: rest
        #readinessProbe: # Note: disable the readiness probe only when cluster bootstrapping
        #  exec:
        #    command:
        #    - "sh"
        #    - "-c"
        #    - "curl -s http://localhost:9200/ | grep 'cluster_uuid' | grep -v '_na_'"  # check whether cluster_uuid is set
        #  periodSeconds: 20
        livenessProbe:
          httpGet:
            port: 9200
            path: /
          initialDelaySeconds: 300
          periodSeconds: 20
        volumeMounts:
        - name: es-data
          mountPath: /usr/share/elasticsearch/data
        envFrom:
        - configMapRef:
            name: elasticsearch-common-config
        - configMapRef:
            name: elasticsearch-master-config
  volumeClaimTemplates:
  - metadata:
      name: es-data
    spec:
      storageClassName: ssd
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 50Gi

storageclass-ssd.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ssd
provisioner: kubernetes.io/gce-pd
parameters:
  type: pd-ssd