Created
October 16, 2013 13:00
-
-
Save CharlieSu/7007305 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
function update_packages () { | |
apt-get update; apt-get upgrade -y | |
apt-get install ntp unzip dstat vim -y | |
} | |
function setup_java7 () { | |
cd /opt | |
wget -q http://yourdomain.com/jdk-7u25-linux-x64.gz -O /opt/jdk-7u25-linux-x64.gz | |
tar -xf jdk-7* | |
mv ./jdk1.7* /opt/jdk1.7.0 | |
ln -s /opt/jdk1.7.0 /opt/java7 | |
} | |
function encrypted_disk_setup () { | |
DEBIAN_FRONTEND=noninteractive apt-get install mdadm cryptsetup -y | |
cat << 'EOF' > /etc/init/crypt.conf | |
author "Charles Sullivan <charles@fullcontact.com>" | |
description "Provision encrypted volumes" | |
start on (local-filesystems ) | |
task | |
script | |
instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" | |
# Test for md127 | |
if ! test -b /dev/md127; then | |
mdadm --verbose --create --run /dev/md127 --level=0 --raid-devices=3 /dev/xvdc /dev/xvdd /dev/xvde | |
sh -c 'echo "`mdadm --detail --scan`" | tee -a /etc/mdadm/mdadm.conf' | |
fi | |
# Encrypt /dev/md127 if needed | |
if ! cryptsetup isLuks /dev/md127; then | |
new_passphrase=$(openssl rand -hex 20) | |
echo -n $new_passphrase > /etc/passphrase | |
echo -n $new_passphrase | cryptsetup -q luksFormat /dev/md127 | |
fi | |
# Start encrypted FS if needed | |
if ! cryptsetup status crypt; then | |
if [ -e /etc/passphrase ]; then | |
cat /etc/passphrase | cryptsetup -q luksOpen /dev/md127 crypt | |
else | |
echo "No passphrase present. Find passphrase for ${instance_id}" | |
stop; exit 1 | |
fi | |
fi | |
# Make FS if required | |
if ! file -s /dev/dm-0 | grep ext4; then | |
mkfs.ext4 /dev/mapper/crypt | |
fi | |
if ! mountpoint -q /mnt/crypt; then | |
mkdir -p /mnt/crypt | |
mount -t ext4 /dev/mapper/crypt /mnt/crypt/ | |
fi | |
## https://bugs.launchpad.net/ubuntu/+source/lvm2/+bug/129488 | |
blockdev --setra 128 /dev/md127 /dev/dm-0 /dev/xvdb /dev/xvdc /dev/xvdd /dev/xvde | |
umount /dev/mapper/crypt | |
blockdev --setra 128 /dev/md127 /dev/dm-0 /dev/xvdb /dev/xvdc /dev/xvdd /dev/xvde | |
mount -o noatime,nodiratime /dev/mapper/crypt /mnt/crypt | |
end script | |
EOF | |
} | |
function setup_elasticsearch () { | |
cd /opt | |
wget -q http://yourdomain.com/elasticsearch/elasticsearch-0.90.5.tar.gz -O /opt/elasticsearch-0.90.5.tar.gz | |
tar xzf elasticsearch-0.90.5.tar.gz | |
ln -s /opt/elasticsearch-0.90.5 /opt/elasticsearch | |
addgroup --system elasticsearch; adduser --system --no-create-home --home /opt/elasticsearch --ingroup elasticsearch elasticsearch | |
export JAVA_HOME=/opt/java7 | |
/opt/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-aws/1.15.0 | |
/opt/elasticsearch/bin/plugin -install lukas-vlcek/bigdesk/2.2.2 | |
/opt/elasticsearch/bin/plugin -install mobz/elasticsearch-head | |
/opt/elasticsearch/bin/plugin -install karmi/elasticsearch-paramedic | |
cat << 'EOF' > /etc/init/elasticsearch-init.conf | |
description "Fix ElasticSearch folder permissions before starting service" | |
start on (local-filesystems and net-device-up and stopped crypt RESULT=ok ) | |
task | |
script | |
mkdir -p /mnt/crypt/elasticsearch/data /mnt/crypt/elasticsearch/work /mnt/crypt/elasticsearch/logs | |
chown -R elasticsearch:elasticsearch /mnt/crypt/elasticsearch | |
end script | |
EOF | |
cat << 'EOF' > /etc/init/elasticsearch.conf | |
description "ElasticSearch Server" | |
start on (local-filesystems and net-device-up and stopped elasticsearch-init RESULT=ok ) | |
stop on shutdown | |
setuid elasticsearch | |
setgid elasticsearch | |
respawn limit 10 5 | |
limit nofile 64000 64000 | |
# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html | |
env JAVA_HOME=/opt/java7 | |
env ES_HEAP_SIZE=8G | |
script | |
set -x | |
# Source Asgard variables | |
wget http://169.254.169.254/latest/user-data -O /tmp/userdata && chmod +x /tmp/userdata | |
. /tmp/userdata | |
/opt/elasticsearch/bin/elasticsearch -f \ | |
-Des.node.name=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) \ | |
-Des.cluster.name=$CLOUD_APP \ | |
-Des.path.conf=/opt/elasticsearch/config | |
end script | |
EOF | |
cat << 'EOF' > /opt/elasticsearch/config/elasticsearch.yml | |
node.master: true | |
node.data: true | |
node.max_local_storage_nodes: 1 | |
bootstrap.mlockall: true | |
path.conf: /opt/elasticsearch/config | |
path.data: /mnt/crypt/elasticsearch/data | |
path.work: /mnt/crypt/elasticsearch/work | |
path.logs: /mnt/crypt/elasticsearch/logs | |
discovery.zen.minimum_master_nodes: 2 | |
discovery.zen.ping.multicast.enabled: true | |
discovery.type: ec2 | |
discovery.ec2.groups: elasticsearch | |
EOF | |
} | |
function setup_copperegg () { | |
cat << 'EOF' > /etc/init/copperegg_install.conf | |
description "Runs the CopperEgg installer once." | |
start on runlevel [2345] | |
stop on runlevel [!2345] | |
task | |
env FLAGFILE=/run/.copperegg | |
script | |
wget http://169.254.169.254/latest/user-data -O /tmp/copperegg_userdata | |
chmod +x /tmp/copperegg_userdata | |
. /tmp/copperegg_userdata | |
if [ -e $FLAGFILE ]; then | |
stop | |
else | |
instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" | |
curl -sk http://API_KEY@api.copperegg.com/rc.sh | RC_TAG=elasticsearch,$CLOUD_APP RC_LABEL=$instance_id RC_LIMITED_ACCESS=y sh | |
touch $FLAGFILE | |
fi | |
end script | |
post-start script | |
sysctl -w kernel.poweroff_cmd=/usr/local/bin/shutdown-hook.sh | |
end script | |
EOF | |
cat << 'EOF' > /usr/local/bin/shutdown-hook.sh | |
#!/bin/bash -x | |
apisrv='api.copperegg.com' | |
apikey='API_KEY' | |
# Remove the system: | |
/usr/local/revealcloud/revealcloud -k ${apikey} -R -m -a ${apisrv} | |
EOF | |
chmod +x /usr/local/bin/shutdown-hook.sh | |
} | |
function setup_stackdriver() { | |
curl -o /etc/apt/sources.list.d/stackdriver.list http://repo.stackdriver.com/precise.list | |
curl --silent https://www.stackdriver.com/RPM-GPG-KEY-stackdriver | apt-key add - | |
apt-get update | |
apt-get install libyajl1 libyajl-dev -y | |
cat << 'EOF' > /etc/default/stackdriver-agent | |
AUTOGENERATE_COLLECTD_CONFIG="yes" | |
STACKDRIVER_API_KEY="API_KEY" | |
EOF | |
DEBIAN_FRONTEND=noninteractive apt-get -yq \ | |
-o Dpkg::Options::=--force-confdef \ | |
-o Dpkg::Options::=--force-confold \ | |
install stackdriver-agent | |
# StackDriver ElasticSearch integration. | |
wget http://yourdomain.com/elasticsearch/elasticsearch-agent.conf -O /opt/stackdriver/collectd/etc/collectd.d/elasticsearch-agent.conf | |
} | |
function setup_ec2toolkit () { | |
cd /opt | |
wget http://yourdomain.com/ec2toolkit/ec2toolkit-1.2-bin.tar.gz | |
tar xzf /opt/ec2toolkit-1.2-bin.tar.gz | |
ln -s /opt/ec2toolkit-1.2 /opt/ec2toolkit | |
} | |
function setup_tagger () { | |
cat << 'EOF' > /etc/init/tagger.conf | |
description "Tag instance" | |
start on (local-filesystems) | |
stop on runlevel [!2345] | |
task | |
script | |
wget http://169.254.169.254/latest/user-data -O /tmp/tagger_userdata && chmod +x /tmp/tagger_userdata | |
. /tmp/tagger_userdata | |
PATH=$PATH:/opt/java7/bin /opt/ec2toolkit-1.2/bin/ec2tag \ | |
-DName=$CLOUD_APP \ | |
-Drole=elasticsearch | |
end script | |
EOF | |
} | |
function unattended_upgrades () { | |
apt-get install unattended-upgrades -y | |
cat << 'EOF' > /etc/apt/apt.conf.d/10periodic | |
APT::Periodic::Enable "1"; | |
APT::Periodic::MaxSize "2048"; | |
APT::Periodic::Update-Package-Lists "1"; | |
APT::Periodic::Download-Upgradeable-Packages "1"; | |
APT::Periodic::Unattended-Upgrade "1"; | |
APT::Periodic::AutocleanInterval "7"; | |
APT::Periodic::Verbose "1"; | |
EOF | |
cat << 'EOF' > /etc/apt/apt.conf.d/50unattended-upgrades | |
Unattended-Upgrade::Allowed-Origins { | |
"${distro_id}:${distro_codename}-security"; | |
"${distro_id}:${distro_codename}-updates"; | |
// "${distro_id}:${distro_codename}-proposed"; | |
// "${distro_id}:${distro_codename}-backports"; | |
}; | |
Unattended-Upgrade::Package-Blacklist { | |
// "vim"; | |
}; | |
EOF | |
} | |
update_packages | |
setup_java7 | |
encrypted_disk_setup | |
setup_elasticsearch | |
setup_copperegg | |
setup_stackdriver | |
setup_ec2toolkit | |
setup_tagger | |
unattended_upgrades |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi Charlie, found this through google. Are you using an encrypted elasticsearch index? what's your experience with that?