Install ElasticSearch in Azure on Ubuntu

1 shell

export esver=1.4.0
sudo apt-get update && sudo apt-get install openjdk-7-jre-headless nginx unzip apache2-utils -y && sudo mkdir /etc/elasticsearch && cd /etc/elasticsearch && sudo wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$esver.zip && sudo unzip elasticsearch-$esver.zip && sudo mv elasticsearch-$esver $esver && echo "" && echo "---" && echo "vi into /etc/elasticsearch/$esver/config/elasticsearch.yml disable 'zen discovery' and enable 'unicast discovery'. Don't worry about the hosts array" && echo "---" && echo "" && sudo touch /etc/init.d/elasticsearch && sudo chmod +x /etc/init.d/elasticsearch && sudo touch /etc/gethosts && sudo chmod +x /etc/gethosts && sudo rm /etc/nginx/sites-enabled/default && sudo touch /etc/nginx/sites-available/elasticsearch && sudo ln -L /etc/nginx/sites-available/elasticsearch /etc/nginx/sites-enabled/elasticsearch && sudo touch /var/log/nginx/kibana.access.log && echo "" && echo "---" && echo "vi into /etc/init.d/elasticsearch and add the contents of the 'elasticsearch' gist" && echo "" && echo "vi into /etc/gethosts and add the contents of the 'gethosts' gist" && echo "" && echo "vi into /etc/nginx/sites-enabled/elasticsearch and add the contents of the 'nginx server' gist" echo "---" && echo "" && sudo htpasswd -c /etc/nginx/conf.d/kibana.htpasswd esuser
echo

2 gethosts

#! /bin/bash
#Build Hosts files
# This assumes your whole cluster lives on the same network subnet. Add your virtual machines to an Azure virtual network

ES_HOSTS=()
IP=()

while getopts a:b:c:f:m:s: option
do
        case "${option}"
        in
                a) IP[0]=${OPTARG};;
                b) IP[1]=${OPTARG};;
                c) IP[2]=${OPTARG};;
                f) MIN_IP_BEFORE_FAIL=$OPTARG;;
                                m) MAX_NODES=$OPTARG;;
                                s) START_IP_AT=$OPTARG;;
        esac
done


FAIL=0
for (( i=$START_IP_AT; i<$(($START_IP_AT + $MAX_NODES)); i++));
do
  if [ $FAIL -ne 1 ]
  then
    c="${IP[0]}.${IP[1]}.${IP[2]}.$i"

    ping -w 1 -c 1 -s 1 $c> /dev/null
    RETVAL=$?

    if [ $RETVAL -eq 0 ]
    then
      ES_HOSTS+=($c)
    else
      if [ $i -gt $MIN_IP_BEFORE_FAIL ]
        then
          let FAIL=1
      fi
    fi
  fi
done

ES_HOSTS_COUNT=${#ES_HOSTS[@]}

ES_HOSTS_STRING=""
for (( i=0; i<$ES_HOSTS_COUNT; i++));
do
  if [ "$(($i + 1))" != "$ES_HOSTS_COUNT" ]
    then ES_HOSTS_STRING="$ES_HOSTS_STRING${ES_HOSTS[$i]},"
    else ES_HOSTS_STRING="$ES_HOSTS_STRING${ES_HOSTS[$i]}"
  fi
done

echo "$ES_HOSTS_STRING"

3 elasticsearch

#! /bin/bash
### BEGIN INIT INFO
# Provides:          elasticsearch
# Required-Start:    $all
# Required-Stop:    $all
# Default-Start:    2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Starts elasticsearch
# Description:      Starts elasticsearch using start-stop-daemon
### END INIT INFO

#Base Params
ES_VER=1.4.0
ES_HOME=/etc/elasticsearch/$ES_VER

#Leave enough memory for the OS to run
ES_MIN_MEM=700mb
ES_MAX_MEM=$ES_MIN_MEM

NAME=elasticsearch
DESC=elasticsearch
DAEMON=$ES_HOME/bin/elasticsearch

PID_FILE=/var/run/$NAME.pid
LOG_DIR=/var/log/$NAME
DATA_DIR=/var/lib/$NAME
WORK_DIR=/tmp/$NAME

CONFIG_FILE=$ES_HOME/config/elasticsearch.yml

#Get the local IP and set subnet config
IP=( $(echo `ifconfig|xargs|awk '{print $7}'|sed -e 's/[a-z]*:/''/'` | grep -o '[0-9]*') )
MIN_IP_BEFORE_FAIL=4
MAX_NODES=20
START_IP_AT=4

set -e

case "$1" in
  start)
    echo -n "Starting $DESC: "

    #Crawl the subnet trolling for nodes
    ES_HOSTS_STRING=$(/etc/gethosts -a ${IP[0]} -b ${IP[1]} -c ${IP[2]} -f $MIN_IP_BEFORE_FAIL -m $MAX_NODES -s $START_IP_AT)

    #Build Options
    DAEMON_OPTS="-p $PID_FILE -Des.config=$CONFIG_FILE -Des.node.name='n_${IP[0]}.${IP[1]}.${IP[2]}.${IP[3]}' -Des.discovery.zen.ping.unicast.hosts='$ES_HOSTS_STRING' -Des.gateway.recover_after_nodes=$ES_HOSTS_COUNT -Des.gateway.expected_nodes=$ES_HOSTS_COUNT -Des.path.home=$ES_HOME -Des.path.logs=$LOG_DIR -Des.path.data=$DATA_DIR -Des.path.work=$WORK_DIR"

    mkdir -p $LOG_DIR $DATA_DIR $WORK_DIR

    if start-stop-daemon --start --pidfile $PID_FILE --startas $DAEMON -- $DAEMON_OPTS
    then
        echo "started."
    else
        echo "failed."
    fi
    ;;
  stop)
    echo -n "Stopping $DESC: "
    if start-stop-daemon --stop --pidfile $PID_FILE
    then
        echo "stopped."
    else
        echo "failed."
    fi
    ;;
  restart|force-reload)
    ${0} stop
    sleep 0.5
    ${0} start
    ;;
  *)
    N=/etc/init.d/$NAME
    echo "Usage: $N {start|stop|restart|force-reload}" >&2
    exit 1
    ;;
esac

exit 0

4 nginx server

server {
  listen                *:80;

  server_name           elasticsearch;
  access_log            /var/log/nginx/kibana.access.log;

  location ~ ^/es/.* {
    rewrite ^/es/(.*) /$1 break;
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;

    auth_basic "Restricted";
    auth_basic_user_file /etc/nginx/conf.d/kibana.htpasswd;
  }
}

Tested on Ubuntu 14.04 and 14.10 in Microsoft Azure.

Create your virtual network and machines
Create a virtual network and add your vm's to the same subnet for the gethosts script to do discovery for you. It's important your multi-node cluster is on a virtual network to enable internal communication over port 9300 for your nodes.

Add port 80 as an endpoint on your vms. I use 80 rather than 9200 because I usually also have Kibana in the same nginx server so basic auth works.

Run shell command
Change the version number of the top line of the shell script gist to the latest release of ElasticSearch. You'll also have to do this in the ES_VER value in the elasticsearch gist.

Run the contents of the shell script gist, set a user password for accessing ElasticSearch when asked at the end.

Insert the contents of created files
Add the contents of the elasticsearch, gethosts and nginx server scripts to their appropriate files.

• elasticearch in /etc/init.d/elasticsearch
• gethosts in /etc/gethosts
• nginx in /etc/nginx/sites-enabled/elasticsearch

Make sure you run sudo service nginx restart after you've updated the nginx server config.

Disable multicast discovery
Open /etc/elasticsearch/$esver/config/elasticsearch.yml and edit the discovery section to disable multicast discovery. It's right near the bottom of the file.

Run ElasticSearch with /etc/init.d/elasticsearch start