Ubuntu 14.04 AWS EC2 cloud-init (Kernel 3.18+NTP+ixgbevf+Docker+Route53) - to be used with ami-870a2fb7 and newer instance types (eg. t2, m3, c3, r3)

gistfile1.sh

#!/bin/bash
set -e

sudo rm -f /etc/update-motd.d/10-help-text
sudo rm -f /etc/update-motd.d/51-cloudguest
sudo rm -f /etc/update-motd.d/91-release-upgrade
echo -e "[sysinfo]\nexclude_sysinfo_plugins = LandscapeLink" | sudo tee /etc/landscape/client.conf

echo deb https://get.docker.com/ubuntu docker main | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
sudo apt-get -qq update
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -qq -y
sudo DEBIAN_FRONTEND=noninteractive apt-get install -qq -y fail2ban pollen htop ssmtp heirloom-mailx mc molly-guard jq zip dkms ntp lxc-docker language-pack-en sysstat nicstat iotop iptraf-ng colordiff

grep -q -F 'unlimited' /etc/init/docker || sudo sed -i "s/respawn/limit memlock unlimited unlimited\nrespawn/" /etc/init/docker.conf

grep -q -F 'noatime' /etc/fstab || sudo sed -i "s/ext4\sdefaults/ext4 noatime,nobarrier,defaults/" /etc/fstab
sudo grep -q -F 'NOPASSWD' /etc/sudoers || echo "%sudo ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers

sudo sed -i 's#ENABLED="false"#ENABLED="true"#g' /etc/default/sysstat

sudo sed -i s/.ubuntu.pool.ntp.org/.amazon.pool.ntp.org\ iburst/g /etc/ntp.conf
echo "tinker panic 0" | cat - /etc/ntp.conf > /tmp/ntp.conf && sudo mv /tmp/ntp.conf /etc/ntp.conf

cd /tmp
sudo curl -s -L -O http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.18.10-vivid/linux-headers-3.18.10-031810_3.18.10-031810.201503241436_all.deb
sudo curl -s -L -O http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.18.10-vivid/linux-headers-3.18.10-031810-generic_3.18.10-031810.201503241436_amd64.deb
sudo curl -s -L -O http://kernel.ubuntu.com/~kernel-ppa/mainline/v3.18.10-vivid/linux-image-3.18.10-031810-generic_3.18.10-031810.201503241436_amd64.deb

sudo DEBIAN_FRONTEND=noninteractive dpkg --force-all -i linux-headers-3.18.10-031810_3.18.10-031810.201503241436_all.deb
sudo DEBIAN_FRONTEND=noninteractive dpkg --force-all -i linux-headers-3.18.10-031810-generic_3.18.10-031810.201503241436_amd64.deb
sudo DEBIAN_FRONTEND=noninteractive dpkg --force-all -i linux-image-3.18.10-031810-generic_3.18.10-031810.201503241436_amd64.deb
sudo update-grub

curl -s -L -O http://downloadmirror.intel.com/18700/eng/ixgbevf-2.16.1.tar.gz
tar -xzf ixgbevf-2.16.1.tar.gz
sudo mv ixgbevf-2.16.1 /usr/src/
sudo tee /usr/src/ixgbevf-2.16.1/dkms.conf <<EOF
PACKAGE_NAME="ixgbevf"
PACKAGE_VERSION="2.16.1"
CLEAN="cd src/; make clean"
MAKE="cd src/; make BUILD_KERNEL=\${kernelver}"
BUILT_MODULE_LOCATION[0]="src/"
BUILT_MODULE_NAME[0]="ixgbevf"
DEST_MODULE_LOCATION[0]="/updates"
DEST_MODULE_NAME[0]="ixgbevf"
AUTOINSTALL="yes"
EOF
sudo dkms add -q -m ixgbevf -v 2.16.1 -k 3.18.10-031810-generic
sudo dkms build -q -m ixgbevf -v 2.16.1 -k 3.18.10-031810-generic
sudo dkms install -q -m ixgbevf -v 2.16.1 -k 3.18.10-031810-generic
sudo update-initramfs -c -k all
echo "options ixgbevf InterruptThrottleRate=1,1,1,1,1,1,1,1" | sudo tee /etc/modprobe.d/ixgbevf.conf

sudo tee /etc/sysctl.d/60-custom.conf <<EOF
# Auto-reboot linux 10 seconds after a kernel panic
kernel.panic = 10
kernel.panic_on_oops = 10
kernel.unknown_nmi_panic = 10
kernel.panic_on_unrecovered_nmi = 10
kernel.panic_on_io_nmi = 10

# Controls whether core dumps will append the PID to the core filename, useful for debugging multi-threaded applications
kernel.core_uses_pid = 1

# Turn on address space randomization - security is super important at BC
kernel.randomize_va_space = 2

vm.swappiness = 0
vm.dirty_ratio = 80
vm.dirty_background_ratio = 5
vm.dirty_expire_centisecs = 12000
vm.overcommit_memory = 1

# ------ VM ------

fs.file-max = 204708
fs.epoll.max_user_instances = 4096
fs.suid_dumpable = 0

# ------ NETWORK SECURITY ------

# Turn on protection for bad icmp error messages
net.ipv4.icmp_ignore_bogus_error_responses = 1

# Turn on syncookies for SYN flood attack protection
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 8096
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2

# Log suspicious packets, such as spoofed, source-routed, and redirect
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.default.log_martians = 1

# Disables these ipv4 features, not very legitimate uses
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0

# ------ NETWORK PERFORMANCE ------

# Netflix 2014 recommendations
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem = 4096 12582912 16777216

# Allow reusing sockets in TIME_WAIT state for new connections
net.ipv4.tcp_tw_reuse = 1

# Socket max connections waiting to get accepted; the listen() backlog.
# Default is 128.
net.core.somaxconn = 4096

# Decrease fin timeout. After telling the client we are closing, how long to wait for a FIN, ACK?
# Default is 60.
net.ipv4.tcp_fin_timeout = 10

# Avoid falling back to slow start after a connection goes idle
# keeps our cwnd large with the keep alive connections
net.ipv4.tcp_slow_start_after_idle = 0
EOF

sudo tee /etc/security/limits.conf <<EOF
* soft nofile 262144
* hard nofile 262144

* soft memlock unlimited
* hard memlock unlimited

* soft nproc 32000
* hard nproc 32000

* soft core 0
* hard core 0
EOF

sudo tee /etc/init.d/custom-tweaks <<EOF
#!/bin/bash

ethtool -K eth0 gro on gso on tso off

if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
   echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
   echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
EOF

sudo chmod +x /etc/init.d/custom-tweaks
sudo update-rc.d custom-tweaks defaults 99

sudo tee /etc/apt/apt.conf.d/10periodic <<EOF
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Download-Upgradeable-Packages "1";
APT::Periodic::AutocleanInterval "7";
APT::Periodic::Unattended-Upgrade "1";
EOF

sudo mkdir -p /opt/bc
curl -o - https://bootstrap.pypa.io/get-pip.py | sudo python2.7
sudo pip install -q awscli

server_name=$(aws ec2 describe-tags --region us-west-2 --filters "Name=resource-id,Values=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)" "Name=tag-key,Values=Name" --query 'Tags[*].Value' --output text)
domain_name=$(aws ec2 describe-tags --region us-west-2 --filters "Name=resource-id,Values=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)" "Name=tag-key,Values=DomainName" --query 'Tags[*].Value' --output text)
local_ipv4=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
zone_id=$(aws route53 --region us-west-2 list-hosted-zones --max-items 1 | jq -r .HostedZones[0].Id)

sudo echo $server_name > /etc/hostname
sudo echo "$local_ipv4 $server_name.$domain_name $server_name" >> /etc/hosts

grep -q -F '$domain_name' /etc/dhcp/dhclient.conf || echo "prepend domain-name \"$domain_name \";" | sudo tee -a /etc/dhcp/dhclient.conf

sudo tee /etc/init.d/vpc-route53 <<EOF
#!/bin/sh
PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin
FQDN=\$(hostname -f)
ZONE_ID=\$(aws route53 --region us-west-2 list-hosted-zones | jq -r '.HostedZones | .[length-1] | .Id')
TTL=300
SELF_META_URL="http://169.254.169.254/latest/meta-data"
PUBLIC_DNS=\$(curl \${SELF_META_URL}/local-hostname 2>/dev/null)

cat << EOT > /tmp/aws_r53_batch.json
{
"Comment": "Assign AWS Public DNS as a CNAME of hostname",
"Changes": [
  {
    "Action": "UPSERT",
    "ResourceRecordSet": {
      "Name": "\${FQDN}.",
      "Type": "CNAME",
      "TTL": \${TTL},
      "ResourceRecords": [
        {
          "Value": "\${PUBLIC_DNS}"
        }
      ]
    }
  }
]
}
EOT

aws route53 change-resource-record-sets --region us-west-2 --hosted-zone-id \${ZONE_ID} --change-batch file:///tmp/aws_r53_batch.json
rm -f /tmp/aws_r53_batch.json
EOF

sudo chmod +x /etc/init.d/vpc-route53
sudo update-rc.d vpc-route53 defaults 99

sudo sed -i "s/HISTSIZE=1000/HISTSIZE=30/" /etc/skel/.bashrc
sudo sed -i "s/HISTFILESIZE=2000/HISTFILESIZE=0/" /etc/skel/.bashrc
sudo sed -i "s/#force_color_prompt=yes/force_color_prompt=yes/" /etc/skel/.bashrc
sudo sed -i s/@\\\\h/@"\$(hostname -f)"/g /etc/skel/.bashrc

sudo sed -i "s/HISTSIZE=1000/HISTSIZE=30/" /root/.bashrc
sudo sed -i "s/HISTFILESIZE=2000/HISTFILESIZE=0/" /root/.bashrc
sudo sed -i "s/#force_color_prompt=yes/force_color_prompt=yes/" /root/.bashrc
sudo sed -i s/@\\\\h/@"\$(hostname -f)"/g /root/.bashrc

sudo sed -i "s/HISTSIZE=1000/HISTSIZE=30/" /home/ubuntu/.bashrc
sudo sed -i "s/HISTFILESIZE=2000/HISTFILESIZE=0/" /home/ubuntu/.bashrc
sudo sed -i "s/#force_color_prompt=yes/force_color_prompt=yes/" /home/ubuntu/.bashrc
sudo sed -i s/@\\\\h/@"\$(hostname -f)"/g /home/ubuntu/.bashrc

sudo sed -i "s/#SULOG_FILE/SULOG_FILE/" /etc/login.defs
sudo sed -i "s/LOG_OK_LOGINS\t\tno/LOG_OK_LOGINS\t\tyes/" /etc/login.defs

sudo wget -O /usr/bin/docker-gc https://raw.githubusercontent.com/spotify/docker-gc/master/docker-gc
sudo chmod 755 /usr/bin/docker-gc

sudo reboot

Make sure you also start the EC2 instance in a VPC and using the following IAM role:

Permissions policy:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "ec2:DescribeTags",
        "ec2:DescribeInstances"
      ],
      "Resource": [
        "*"
      ]
    },
    {
      "Effect": "Allow",
      "Action": [
        "route53:ChangeResourceRecordSets",
        "route53:GetHostedZone",
        "route53:List*"
      ],
      "Resource": [
        "*"
      ]
    }
  ]
}

Trust policy:

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

For example (Ansible):

    - name: provision statsd instances
      ec2:
        region: "{{ ec2_region }}"
        vpc_subnet_id: "{{ subnet_maps['10.0.1.0/24'] }}"
        keypair: "{{ ec2_keypair }}"
        group_id: ["{{ default_group.group_id }}", "{{ statsd_group.group_id }}"]
        image: "{{ ec2_statsd_image|default('ami-870a2fb7', true) }}"
        instance_type: "{{ ec2_statsd_instance_type|default('t2.small', true) }}"
        instance_profile_name: AWSLimitedAccess
        volumes:
          - device_name: /dev/sda1
            device_type: gp2
            volume_size: 30
        user_data: "#include https://gist.githubusercontent.com/analytically/1a6bdcfa711474d7ba24/raw"
        instance_tags:
          Name: "{{ 'statsd' + item }}"
          Class: statsd
          DomainName: "{{ internal_domain_name }}"
        exact_count: 1
        count_tag:
          Name: "{{ 'statsd' + item }}"
        wait: yes
      register: statsd
      with_sequence: count=1
      tags:
        - statsd

Why use sudo during cloud-init? It runs as root already...

Would be great if this was annotated ;)