ElasticSearch Bootstrap

0-dependencies.sh

# Environment
#
#     export HOSTNAME=...
#     export CLUSTER=...
#     export RACK=...
#     export ESDIR=
#     export AWS_ACCESS_KEY=...
#     export AWS_SECRET_KEY=...

# Packages
sudo yum install -y make gcc gcc-c++ git rubygems screen compat-expat {ruby,ruby19,python27,openssl,sqlite,libxslt,libevent,expat}{,-devel} ncdu iptraf mdadm xfsprogs

# Security limits
echo 'ec2-user soft nofile 65535' | sudo tee -a /etc/security/limits.conf
echo 'ec2-user hard nofile 65535' | sudo tee -a /etc/security/limits.conf
# Does this require a restart?

# System-wise file descriptor limit
echo 'fs.file-max = 100000' | sudo tee -a /etc/sysctl.conf

# Make the hostname rebootable
sudo sed -i "s#HOSTNAME=localhost.localdomain#HOSTNAME=`hostname`#" /etc/sysconfig/network

# And unlimited memory locking
echo "ec2-user soft memlock unlimited" | sudo tee -a /etc/security/limits.conf
echo "ec2-user hard memlock unlimited" | sudo tee -a /etc/security/limits.conf
# Don't limit the number of processes
echo "ec2-user soft nproc unlimited" | sudo tee -a /etc/security/limits.conf
echo "ec2-user hard nproc unlimited" | sudo tee -a /etc/security/limits.conf

1-python.sh

# Setuptools
cd ~/ && curl -O http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz
tar xf setuptools-0.6c11.tar.gz && cd setuptools-0.6c11 && sudo python27 setup.py install

# Pip
cd ~/ && curl -O http://pypi.python.org/packages/source/p/pip/pip-1.1.tar.gz
tar xf pip-1.1.tar.gz && cd pip-1.1 && sudo python27 setup.py install

# Cleanup
cd ~/ && sudo rm -rdf setuptools-0.6c11* pip-1.1*

# Some python packages
sudo pip-2.7 install --upgrade lxml boto pyyaml psutil shovel hiredis argparse pyopenssl simplejson python-dateutil redis

2-raid.sh

# Raid the ephemeral devices together and mount at /mnt
sudo umount /media/ephemeral0

# How many devices, and which
device_count=`sudo ls /dev/xvd[b-z] | wc -l`
devices=`sudo ls /dev/xvd[b-z]`
raid_level=0

# Create it
yes | sudo mdadm /dev/md0 --create --run --assume-clean \
    --level raid${raid_level} --raid-devices=${device_count} ${devices}
sudo mdadm --detail --scan | sudo tee -a /etc/mdadm.conf

# Edit /etc/fstab and replace the last line with:
sudo sed -i '$ d' /etc/fstab
echo '/dev/md0    /media/raid ext4    defaults        0   0' | sudo tee -a /etc/fstab

# Make an ext4 on /dev/md0
sudo mkfs.ext4 /dev/md0

# And load it
sudo mkdir /media/raid
sudo mount /media/raid

sudo chmod a+rwx /media/raid

# Needs this in order to be rebootable
sudo dracut --mdadmconf --force /boot/initramfs-$(uname -r).img $(uname -r)

3-ncdu.sh

# It's a useful little tool
cd && curl -OL http://dev.yorhel.nl/download/ncdu-1.9.tar.gz
tar xf ncdu-1.9.tar.gz 
cd ncdu-1.9
./configure && make -j2 && sudo make install
cd && rm -rdf ncdu*

5-elasticsearch.sh

export ES_VERSION=0.20.1

# Download
curl -OL http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.zip

# Unpack, link, cleanup
sudo unzip elasticsearch-$ES_VERSION.zip -d /usr/local/elasticsearch
sudo ln -s /usr/local/elasticsearch/elasticsearch-{$ES_VERSION,latest}
rm elasticsearch-$ES_VERSION.zip

# Some profile stuff
echo 'export ESHOME=/usr/local/elasticsearch/elasticsearch-latest/' >> ~/.bash_profile
echo 'export PATH=$PATH:/usr/local/elasticsearch/elasticsearch-latest/bin/service/' >> ~/.bash_profile
echo "export ES_HEAP_SIZE=8192" | tee -a ~/.bash_profile
source ~/.bash_profile

# Install the AWS cloud plugin
cd $ESHOME
sudo bin/plugin -install elasticsearch/elasticsearch-cloud-aws/1.4.0
sudo bin/plugin -install karmi/elasticsearch-paramedic

# Install this as a service
mkdir ~/git
cd ~/git && git clone https://github.com/elasticsearch/elasticsearch-servicewrapper.git
cp -r elasticsearch-servicewrapper/service $ESHOME/bin/

# Make the service honor ES_JAVA_OPTS
echo 'wrapper.java.additional.10=%ES_JAVA_OPTS%' | sudo tee -a /usr/local/elasticsearch/elasticsearch-latest/bin/service/elasticsearch.conf

6-elasticconfig.sh

source ~/.bash_profile

export ES_CONFIG=$ESHOME/config/elasticsearch.yml

# Set the node name
sudo sed -i "s;# node.name: \"Franz Kafka\";node.name: \"$HOSTNAME\";" $ES_CONFIG
sudo sed -i "s;# cluster.name: elasticsearch;cluster.name: $CLUSTER;" $ES_CONFIG
sudo sed -i "s;# node.rack: rack314;node.rack: $RACK;" $ES_CONFIG
sudo sed -i 's;gateway.recover_after_time: 2m;gateway.recover_after_time: 10m;' $ES_CONFIG

export ES_LOGGING=$ESHOME/config/logging.yml
sudo sed -i 's;#gateway: DEBUG;gateway: DEBUG;' $ES_LOGGING
sudo sed -i 's;#index.gateway: DEBUG;index.gateway: DEBUG;' $ES_LOGGING
sudo sed -i 's;#indices.recovery: DEBUG;indices.recovery: DEBUG;' $ES_LOGGING
sudo sed -i 's;#discovery: TRACE;discovery: DEBUG;' $ES_LOGGING

export ES_LOGS_DIR="$ESDIR/logs"; mkdir -p $ES_LOGS_DIR
export ES_DATA_DIR="$ESDIR/data"; mkdir -p $ES_DATA_DIR
export ES_WORK_DIR="$ESDIR/work"; mkdir -p $ES_WORK_DIR

sudo sed -i "s;# path.data: /path/to/data$;path.data: $ES_DATA_DIR;" $ES_CONFIG
sudo sed -i "s;# path.work: /path/to/work$;path.work: $ES_WORK_DIR;" $ES_CONFIG
sudo sed -i "s;# path.logs: /path/to/logs$;path.logs: $ES_LOGS_DIR;" $ES_CONFIG

# Allow it to lock memory
sudo sed -i "s;# bootstrap.mlockall: true;bootstrap.mlockall: true;" $ES_CONFIG

# And allow large uploads
sudo sed -i "s;# http.max_content_length: 100mb;http.max_content_length: 1gb;" $ES_CONFIG

# Gateway configuration
sudo sed -i "s;# gateway.type: local;gateway.type: local;" $ES_CONFIG
sudo sed -i "s;# gateway.recover_after_time: 5m;gateway.recover_after_time: 2m;" $ES_CONFIG
sudo sed -i "s;# discovery.zen.minimum_master_nodes: 1;discovery.zen.minimum_master_nodes: 6;" $ES_CONFIG

# Discovery
echo "cloud:
    aws:
        access_key: $AWS_ACCESS_KEY
        secret_key: $AWS_SECRET_KEY

discovery:
    type: ec2
    # This should be either 'production' or 'dev'
    ec2.tag.search: production" | sudo tee -a $ES_CONFIG

echo "threadpool:
    search:
        type: blocking
        min: 1
        size: 8
        wait_time: 30s" | sudo tee -a sudo tee -a $ES_CONFIG

# And we /don't/ want to dump the heap on failure
sudo sed -i s'#-XX:+HeapDumpOnOutOfMemoryError#-XX:-HeapDumpOnOutOfMemoryError#g' /usr/local/elasticsearch/elasticsearch-latest/bin/service/elasticsearch.conf

# Use TCP Keep-Alive
echo "network.tcp.keep_alive: true" | sudo tee -a /usr/local/elasticsearch/elasticsearch-latest/config/elasticsearch.yml 

7-zabbix.sh

sudo rpm -Uvh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-7.noarch.rpm
sudo yum -y install zabbix-agent

sudo rm /etc/zabbix/zabbix_agentd.conf
echo "PidFile=/var/run/zabbix/zabbix_agentd.pid
LogFile=/var/log/zabbix/zabbix_agentd.log

# Require for "active" style checks
EnableRemoteCommands=1

# AWS web proxy
Server=10.99.39.132

# Hostname fed into script
Hostname=$HOSTNAME

# Some types of checks can take awhile. 
Timeout=30" | sudo tee /etc/zabbix/zabbix_agentd.conf

# Make some directories writeable
sudo mkdir -p /var/lock/subsys/zabbix
sudo chmod -R a+rw /var/log/zabbix /var/lock/ /var/lock/subsys/zabbix /var/run/zabbix/

# Restart zabbix can take awhile
/etc/init.d/zabbix-agent stop
/etc/init.d/zabbix-agent stop
/etc/init.d/zabbix-agent start

9-reboot.sh

sudo reboot