Docker experimental networking setup on AWS

bootstrap-aws-docker-networking

#!/usr/bin/env bashsh-0

region=eu-central-1
ami=ami-accff2b1  # Ubuntu Server 14.04 LTS (HVM), SSD Volume Type
docker_binary_url='http://rabe.io/docker-1.9.0-dev'
docker_binary_sha=ec1815620c58248defd8540dcdeccd113df4d9f5  # 5ceff3f experimental linux/amd64
# docker_binary_url='https://experimental.docker.com/builds/Linux/x86_64/docker-latest'

function e() {
  aws ec2 --region "$region" --output json "$@"
}

function E() {
  CMD aws ec2 --region "$region" --output json "$@"
}

function wait_for_state() {
  local state=$1
  shift
  local instance

  for instance ; do
    MSG "Waiting for $instance to be $state ..."
    while [[ "$(e describe-instances --instance-ids "$instance" --query 'Reservations[0].Instances[0].State.Name' --output text)" != "$state" ]] ; do
      sleep 2
    done
    MSG "Instance $instance is now $state"
  done
}

MSG "Terminating instances"
instances=$(e describe-tags --filters Name=key,Values=DockerNet --query 'Tags[*].ResourceId' --output text)
for instance in $instances ; do
  E terminate-instances --instance-ids "$instance" --query 'TerminatingInstances[0].CurrentState'
  E delete-tags --resources "$instance" --tags Key=DockerNet
done
wait_for_state terminated $instances

MSG "Re-creating security group"
E delete-security-group --group-name docker-net-sg || true

# exit 1

E create-security-group --group-name docker-net-sg --description "Docker networking security group"
E authorize-security-group-ingress --group-name docker-net-sg --protocol tcp --port 22 --cidr 0.0.0.0/0
E authorize-security-group-ingress --group-name docker-net-sg --protocol icmp --port -1 --source-group docker-net-sg
# E authorize-security-group-ingress --group-name docker-net-sg --protocol tcp --port 2379 --source-group docker-net-sg
E authorize-security-group-ingress --group-name docker-net-sg --protocol tcp --port 8300 --source-group docker-net-sg
E authorize-security-group-ingress --group-name docker-net-sg --protocol tcp --port 8301 --source-group docker-net-sg
E authorize-security-group-ingress --group-name docker-net-sg --protocol tcp --port 7946 --source-group docker-net-sg

MSG "Re-creating key pair"
keyfile="$DIR"/docker-net-key.pem
CMD rm -f "$keyfile"
E delete-key-pair --key-name docker-net-key || true
E create-key-pair --key-name docker-net-key --query 'KeyMaterial' --output text > "$keyfile"
CMD chmod 0400 "$keyfile"

MSG "Running instances"
instance=$(E run-instances --image-id "$ami" --count 1 --instance-type t2.micro --key-name docker-net-key --security-groups docker-net-sg --query 'Instances[0].InstanceId' --output text)
E create-tags --resources "$instance" --tags Key=DockerNet,Value=1
instance=$(E run-instances --image-id "$ami" --count 1 --instance-type t2.micro --key-name docker-net-key --security-groups docker-net-sg --query 'Instances[0].InstanceId' --output text)
E create-tags --resources "$instance" --tags Key=DockerNet,Value=2

function ssh_() {
  local ip=$1
  shift
  ssh -o ServerAliveInterval=100 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$keyfile" ubuntu@"$ip" "$@"
}

function SSH() {
  local ip=$1
  shift
  CMD ssh -o ServerAliveInterval=100 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$keyfile" ubuntu@"$ip" "$@"
}

function setup_box_a() {
  local num=$1
  local instance=$(e describe-tags --filters Name=key,Values=DockerNet Name=value,Values=$num --query 'Tags[*].ResourceId' --output text)
  local ip=$(e describe-instances --instance-ids "$instance" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
  MSG "Set up box $num ($instance ; $ip) (A)"

  MSG "Waiting for SSH connection ..."
  while ! ssh_ "$ip" true ; do
    sleep 2
  done
  SSH "$ip" sudo apt-get update -qq
  SSH "$ip" 'sudo debconf-set-selections <<< "grub grub/update_grub_changeprompt_threeway select install_new"'
  SSH "$ip" sudo DEBIAN_FRONTEND=noninteractive apt-get install -qqy linux-image-generic-lts-utopic
  SSH "$ip" sudo DEBIAN_FRONTEND=noninteractive apt-get install -qqy unzip

  E reboot-instances --instance-ids $instance
}

instances=$(e describe-tags --filters Name=key,Values=DockerNet --query 'Tags[*].ResourceId' --output text)
wait_for_state running $instances
CMD sleep 3
setup_box_a 1
setup_box_a 2


function setup_box_b() {
  local num=$1
  local instance=$(e describe-tags --filters Name=key,Values=DockerNet Name=value,Values=$num --query 'Tags[*].ResourceId' --output text)
  local ip=$(e describe-instances --instance-ids "$instance" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
  MSG "Set up box $num ($instance ; $ip) (B)"

  MSG "Waiting for SSH connection ..."
  while ! ssh_ "$ip" true ; do
    sleep 2
  done
  # SSH "$ip" uname -a
  SSH "$ip" 'curl -sSL https://experimental.docker.com/ | sh'
  SSH "$ip" sudo usermod -aG docker ubuntu
  SSH "$ip" sudo stop docker

  # CMD scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$keyfile" "$DIR/docker/bundles/1.9.0-dev/binary/docker-1.9.0-dev" ubuntu@"$ip":~/docker
  SSH "$ip" curl -o docker "$docker_binary_url"
  SSH "$ip" "shasum -c <(echo '$docker_binary_sha  docker') > /dev/null 2>&1"
  SSH "$ip" sudo install docker /usr/local/bin/docker

  SSH "$ip" curl -OL https://dl.bintray.com/mitchellh/consul/0.5.2_linux_amd64.zip
  SSH "$ip" unzip 0.5.2_linux_amd64.zip
  SSH "$ip" sudo mv consul /usr/local/bin
  SSH "$ip" rm 0.5.2_linux_amd64.zip
}

wait_for_state running $instances
CMD sleep 3
setup_box_b 1
setup_box_b 2

instance_1=$(e describe-tags --filters Name=key,Values=DockerNet Name=value,Values=1 --query 'Tags[*].ResourceId' --output text)
ip_1=$(e describe-instances --instance-ids "$instance_1" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
internal_1=$(ssh_ "$ip_1" 'ip a s dev eth0 | fgrep "  inet " | cut -d " " -f 6 | cut -d / -f 1')

instance_2=$(e describe-tags --filters Name=key,Values=DockerNet Name=value,Values=2 --query 'Tags[*].ResourceId' --output text)
ip_2=$(e describe-instances --instance-ids "$instance_2" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text)
internal_2=$(ssh_ "$ip_2" 'ip a s dev eth0 | fgrep "  inet " | cut -d " " -f 6 | cut -d / -f 1')

MSG "Start Consul"
SSH "$ip_1" 'nohup consul agent -server -bootstrap -data-dir /tmp/consul -bind '"$(QUOTED "$internal_1")"' >> /tmp/consul-agent.log 2>&1 &'
CMD sleep 2
SSH "$ip_2" 'nohup consul agent -data-dir /tmp/consul -bind '"$(QUOTED "$internal_2")"' >> /tmp/consul-agent.log 2>&1 &'
# Optional, joining can also be done by --label=com.docker.network.driver.overlay.neighbor_ip=x.x.x.x ; see https://github.com/docker/libnetwork/blob/d0c49908/drivers/overlay/ov_serf.go#L98
CMD sleep 2
SSH "$ip_2" 'consul join '"$(QUOTED "$internal_1")"''
CMD sleep 2

MSG "Start Docker"
SSH "$ip_1" 'sudo nohup docker daemon --default-network=overlay:multihost --kv-store=consul:localhost:8500 >> /tmp/docker-daemon.log 2>&1 &'
# # Optional, already joined above
# SSH "$ip_2" 'sudo nohup docker daemon --kv-store=consul:localhost:8500 --label=com.docker.network.driver.overlay.bind_interface=eth0 --label=com.docker.network.driver.overlay.neighbor_ip='"$(QUOTED "$internal_1")"' >> /tmp/docker-daemon.log 2>&1 &'
SSH "$ip_2" 'sudo nohup docker daemon --default-network=overlay:multihost --kv-store=consul:localhost:8500 >> /tmp/docker-daemon.log 2>&1 &'

E describe-instances --instance-ids $instances --query 'Reservations[*].Instances[*].[Tags[0].Value,InstanceId,PublicIpAddress,PrivateIpAddress]' --output text | sort