Configuring Es 7 cluster on containers on GCE Managed Instance Groups (https://medium.com/@mocobeta/elasticsearch-7-%E3%82%AF%E3%83%A9%E3%82%B9%E3%82%BF%E3%82%92-containers-on-gce-%E3%81%A7%E7%AB%8B%E3%81%A1%E4%B8%8A%E3%81%92%E3%82%8B-%E3%81%9D%E3%81%AE2-managed-instance-group-%E7%B7%A8-fec5c8587462)

commands-mig.txt

#
# variables
#
export PROJECT=$(gcloud config get-value project)
export REGISTRY_HOST=asia.gcr.io
export CONTAINER_IMAGE=my-es-7
export REGION=asia-northeast1
export ZONE_1=asia-northeast1-a
export ZONE_2=asia-northeast1-b
export ZONE_3=asia-northeast1-c
export TEMPLATE_MASTER_1=es-master1-template
export TEMPLATE_MASTER_2=es-master2-template
export TEMPLATE_MASTER_3=es-master3-template
export TEMPLATE_DATA=es-data-template
export IG_MASTER_1=es-master1-group
export IG_MASTER_2=es-master2-group
export IG_MASTER_3=es-master3-group
export IG_DATA=es-data-group
export MASTER_DISK_SIZE=50
export DATA_DISK_SIZE=100
export DISK_TYPE=pd-ssd
export ES_CLUSTER=my-es-cluster
export ES_MASTER_NODE_1=es-master-1
export ES_MASTER_NODE_2=es-master-2
export ES_MASTER_NODE_3=es-master-3
export INITIAL_MASTER_NODES=${ES_MASTER_NODE_1},${ES_MASTER_NODE_2},${ES_MASTER_NODE_3}
export HEALTH_CHECK=es-check

#
# building image
#
chmod 755 entrypoint.sh
docker build -q --tag ${CONTAINER_IMAGE}:latest --no-cache .
docker tag ${CONTAINER_IMAGE}:latest ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:latest
docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}
gcloud container images list --repository ${REGISTRY_HOST}/${PROJECT}

#
# create templates
#
gcloud compute instance-templates create-with-container ${TEMPLATE_MASTER_1} \
--metadata-from-file startup-script=instance-setup.sh \
--boot-disk-size ${MASTER_DISK_SIZE} \
--boot-disk-type ${DISK_TYPE} \
--container-privileged \
--container-image ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} \
--container-env "^:^cluster.name=${ES_CLUSTER}:node.name=${ES_MASTER_NODE_1}:node.master=true:node.data=false:cluster.initial_master_nodes=${INITIAL_MASTER_NODES}:cloud.gce.project_id=${PROJECT}:cloud.gce.zone=${ZONE_1},${ZONE_2},${ZONE_3}" \
--scopes default,compute-rw

gcloud compute instance-templates create-with-container ${TEMPLATE_MASTER_2} \
--metadata-from-file startup-script=instance-setup.sh \
--boot-disk-size ${MASTER_DISK_SIZE} \
--boot-disk-type ${DISK_TYPE} \
--container-privileged \
--container-image ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} \
--container-env "^:^cluster.name=${ES_CLUSTER}:node.name=${ES_MASTER_NODE_2}:node.master=true:node.data=false:cluster.initial_master_nodes=${INITIAL_MASTER_NODES}:cloud.gce.project_id=${PROJECT}:cloud.gce.zone=${ZONE_1},${ZONE_2},${ZONE_3}" \
--scopes default,compute-rw

gcloud compute instance-templates create-with-container ${TEMPLATE_MASTER_3} \
--metadata-from-file startup-script=instance-setup.sh \
--boot-disk-size ${MASTER_DISK_SIZE} \
--boot-disk-type ${DISK_TYPE} \
--container-privileged \
--container-image ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} \
--container-env "^:^cluster.name=${ES_CLUSTER}:node.name=${ES_MASTER_NODE_3}:node.master=true:node.data=false:cluster.initial_master_nodes=${INITIAL_MASTER_NODES}:cloud.gce.project_id=${PROJECT}:cloud.gce.zone=${ZONE_1},${ZONE_2},${ZONE_3}" \
--scopes default,compute-rw

gcloud compute instance-templates create-with-container ${TEMPLATE_DATA} \
--metadata-from-file startup-script=instance-setup.sh \
--boot-disk-size ${DATA_DISK_SIZE} \
--boot-disk-type ${DISK_TYPE} \
--container-privileged \
--container-image ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} \
--container-env "^:^cluster.name=${ES_CLUSTER}:node.master=false:node.data=true:cluster.initial_master_nodes=${INITIAL_MASTER_NODES}:cloud.gce.project_id=${PROJECT}:cloud.gce.zone=${ZONE_1},${ZONE_2},${ZONE_3}" \
--scopes default,compute-rw

#
# startup instance groups
#
gcloud compute instance-groups managed create ${IG_MASTER_1} \
--zones ${ZONE_1} \
--size 1 \
--template ${TEMPLATE_MASTER_1}

gcloud compute instance-groups managed create ${IG_MASTER_2} \
--zones ${ZONE_2} \
--size 1 \
--template ${TEMPLATE_MASTER_2}

gcloud compute instance-groups managed create ${IG_MASTER_3} \
--zones ${ZONE_3} \
--size 1 \
--template ${TEMPLATE_MASTER_3}

gcloud compute instance-groups managed create ${IG_DATA} \
--zones ${ZONE_1},${ZONE_2},${ZONE_3} \
--size 3 \
--template ${TEMPLATE_DATA} 

#
# create health check
#
gcloud compute health-checks create http ${HEALTH_CHECK} --port 9200 \
--check-interval 20s \
--healthy-threshold 1 \
--timeout 5s \
--unhealthy-threshold 3

gcloud compute firewall-rules create allow-health-check \
--allow tcp:9200 \
--source-ranges 130.211.0.0/22,35.191.0.0/16 \
--network default

#
# update instance groups
#
gcloud compute instance-groups managed update ${IG_MASTER_1} \
--region ${REGION} \
--health-check ${HEALTH_CHECK} \
--initial-delay 300

gcloud compute instance-groups managed update ${IG_MASTER_2} \
--region ${REGION} \
--health-check ${HEALTH_CHECK} \
--initial-delay 300

gcloud compute instance-groups managed update ${IG_MASTER_3} \
--region ${REGION} \
--health-check ${HEALTH_CHECK} \
--initial-delay 300

gcloud compute instance-groups managed update ${IG_DATA} \
--region ${REGION} \
--health-check ${HEALTH_CHECK} \
--initial-delay 300


#
# list instances
#
gcloud compute instance-groups managed list-instances ${IG_MASTER_1} --region ${REGION}
gcloud compute instance-groups managed list-instances ${IG_MASTER_2} --region ${REGION}
gcloud compute instance-groups managed list-instances ${IG_MASTER_3} --region ${REGION}
gcloud compute instance-groups managed list-instances ${IG_DATA} --region ${REGION}

#
# delete instance groups
#
gcloud compute instance-groups managed delete ${IG_MASTER_1} --quiet --region ${REGION}
gcloud compute instance-groups managed delete ${IG_MASTER_2} --quiet --region ${REGION}
gcloud compute instance-groups managed delete ${IG_MASTER_3} --quiet --region ${REGION}
gcloud compute instance-groups managed delete ${IG_DATA} --quiet --region ${REGION}

#
# delete templates
#
gcloud compute instance-templates delete --quiet ${TEMPLATE_MASTER_1}
gcloud compute instance-templates delete --quiet ${TEMPLATE_MASTER_2}
gcloud compute instance-templates delete --quiet ${TEMPLATE_MASTER_3}
gcloud compute instance-templates delete --quiet ${TEMPLATE_DATA}

######################################
# commands for veryfing auto healing #
######################################
host=<es node's internal ip>

curl -XPUT ${host}:9200/testidx -HContent-type:application/json --data-binary '
{"settings": {"index": {"number_of_shards": 1, "number_of_replicas": 2}}}
'
curl ${host}:9200/testidx?pretty

for i in {1..100}; do curl -XPOST ${host}:9200/testidx/_doc -HContent-type:application/json --data '{"data":"test"}'; done
curl "${host}:9200/testidx/_search?size=0&pretty"

curl ${host}:9200/_cluster/health?pretty
curl ${host}:9200/_nodes/stats/indices/docs?pretty

Dockerfile

# base image
FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0

# PATH
ENV PATH /usr/share/elasticsearch/bin:$PATH

USER elasticsearch

# copy configuration file
COPY elasticsearch.yml /usr/share/elasticsearch/config/

# install plugins
RUN elasticsearch-plugin install discovery-gce --batch
RUN elasticsearch-plugin install analysis-kuromoji
RUN elasticsearch-plugin install analysis-icu

# switch user to root
USER root

# system configuration
# https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html
RUN mkdir -p /etc/systemd/system/elasticsearch.service.d/
RUN touch /etc/systemd/system/elasticsearch.service.d/override.conf
RUN echo "[Service]" >> /etc/systemd/system/elasticsearch.service.d/override.conf
RUN echo "LimitMEMLOCK=infinity" >> /etc/systemd/system/elasticsearch.service.d/override.conf

COPY entrypoint.sh .
ENTRYPOINT ["./entrypoint.sh"]

elasticsearch.yml

cluster.name: es-test-cluster
#node.name: es-test-node-1
#node.master: true
#node.data: true

bootstrap.memory_lock: true
network.host: 0.0.0.0
network.bind_host: 0.0.0.0

discovery.seed_providers: gce

xpack.license.self_generated.type: basic
xpack.security.enabled: false
xpack.monitoring.enabled: false
xpack.monitoring.collection.enabled: false
xpack.sql.enabled: false
xpack.watcher.enabled: false
xpack.ml.enabled: false

entrypoint.sh

#! /bin/bash

# system configuration
# https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html
ulimit -n 65536
ulimit -l unlimited

# change owner of data directory to avoid access denied exception
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data

# start the elasticsearch node 
/usr/local/bin/docker-entrypoint.sh eswrapper

instance-setup.sh

#!/bin/bash

# increase mmap count
# https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
sysctl -w vm.max_map_count=262144
echo "vm.max_map_count = 262144" >> /etc/sysctl.d/99-sysctl.conf

# increase resource limits
# https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#systemd
ulimit -n 65536
ulimit -l unlimited
echo "root * nofile 65536" >> /etc/security/limits.conf
echo "root * memlock unlimited" >> /etc/security/limits.conf