Elasticsearch Cluster Setup on AWS

ec2-es-run.sh

# !/bin/sh
set -e

# Get the amount of available memory using some hacky python
MAXMEM=$(python -c "import os; print('%dg' % (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024 ** 3)))")

# Define the Elasticsearch java options
export ES_JAVA_OPTS="-Xms$MAXMEM -Xmx$MAXMEM"

# Increase memory setting for Elasticsearch.
sudo sysctl -w vm.max_map_count=262144

# Run elasticsearch
./ES642/bin/elasticsearch

ec2-es-setup.sh

# !/bin/sh
# Script to setup Elasticsearch on an Ubuntu EC2 instance as part of a cluster.
# Usage: ./<script.sh>

set -e

clustername=elasticsearch-cluster
esdir="$HOME/ES642"
esdatadir="$HOME/ESDATA"
eslogdir="$HOME/ESLOGS"
cnf="$esdir/config/elasticsearch.yml"

# Increase memory setting for Elasticsearch.
sudo sysctl -w vm.max_map_count=262144

# Remove, re-download, unzip Elasticsearch source code.
rm -rf $esdir
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.4.2.tar.gz
tar xvf elasticsearch-6.4.2.tar.gz
rm elasticsearch-6.4.2.tar.gz
mv elasticsearch-6.4.2 $esdir

# Install/update JVM.
sudo apt-get update -y
sudo apt-get install -y default-jre htop
sudo apt-get install -y python

# Build a simple config file line-by-line
echo "" > $cnf
echo "cluster.name: $clustername" >> $cnf
echo "node.name: $(cat /etc/hostname)" >> $cnf
echo "path.data: $esdatadir" >> $cnf
echo "path.logs: $eslogdir" >> $cnf
echo "network.host: 0.0.0.0" >> $cnf
echo "action.destructive_requires_name: true" >> $cnf
echo "http.cors.enabled: true" >> $cnf
echo "http.cors.allow-origin: /(null)|(https?:\/\/localhost(:[0-9]+)?)/" >> $cnf

# Specify the discovery protocol to use the discovery-ec2 plugin.
# Note: to get ec2 discovery working, you must create an IAM role with 
# EC2ReadOnlyAccess policy and attach it to the EC2 instances in your cluster..
echo "discovery.zen.hosts_provider: ec2" >> $cnf
bash $esdir/bin/elasticsearch-plugin install -b discovery-ec2

# Print useful information about the configuration.
echo "----"
cat $cnf
echo "----"
which java
java -version
echo "----"
echo "Done"