CoreOS user-data file for a cluster running Progrium's service discovery tools (consul, registrator, and ambassadord)

user-data

#cloud-config

coreos:
  etcd:
    # generate a new token for each unique cluster from https://discovery.etcd.io/new
    # WARNING: replace each time you 'vagrant destroy'
    discovery: https://discovery.etcd.io/new
    addr: $public_ipv4:4001
    peer-addr: $public_ipv4:7001
  fleet:
    public-ip: $public_ipv4
  units:
    - name: etcd.service
      command: start
    - name: fleet.service
      command: start
    - name: docker-tcp.socket
      command: start
      enable: true
      content: |
        [Unit]
        Description=Docker Socket for the API

        [Socket]
        ListenStream=2375
        Service=docker.service
        BindIPv6Only=both

        [Install]
        WantedBy=sockets.target
    - name: consul.service
      command: start
      enable: true
      content: |
        [Unit]
        Description=Consul
        After=fleet.service docker.service
        Requires=docker.service

        [Service]
        TimeoutStartSec=0
        Restart=always
        EnvironmentFile=/etc/environment
        ExecStartPre=-/usr/bin/docker kill consul
        ExecStartPre=-/usr/bin/docker rm consul
        ExecStartPre=/usr/bin/docker pull ah45/consul
        ExecStart=/bin/bash -c "\
          /usr/bin/docker run \
          --name consul \
          -h %H \
          -v /var/run/consul:/data \
          -p ${COREOS_PRIVATE_IPV4}:8300:8300 \
          -p ${COREOS_PRIVATE_IPV4}:8301:8301 \
          -p ${COREOS_PRIVATE_IPV4}:8301:8301/udp \
          -p ${COREOS_PRIVATE_IPV4}:8302:8302 \
          -p ${COREOS_PRIVATE_IPV4}:8302:8302/udp \
          -p ${COREOS_PRIVATE_IPV4}:8400:8400 \
          -p ${COREOS_PRIVATE_IPV4}:8500:8500 \
          -p ${COREOS_PUBLIC_IPV4}:53:53/udp \
          ah45/consul \
          -peers `fleetctl list-machines -fields=ip -no-legend | paste -sd ','` \
          -server \
          -advertise ${COREOS_PRIVATE_IPV4} \
          -bootstrap-expect 3"
        ExecStop=/usr/bin/docker kill consul

        [Install]
        WantedBy=multi-user.target
    - name: registrator.service
      command: start
      enable: true
      content: |
        [Unit]
        Description=Registrator
        After=consul.service
        Requires=consul.service

        [Service]
        TimeoutStartSec=0
        Restart=always
        EnvironmentFile=/etc/environment
        ExecStartPre=-/usr/bin/docker kill registrator
        ExecStartPre=-/usr/bin/docker rm registrator
        ExecStartPre=/usr/bin/docker pull progrium/registrator
        ExecStart=/usr/bin/docker run \
          --name registrator \
          -h %H \
          -v /var/run/docker.sock:/tmp/docker.sock \
          --link consul:consul \
          progrium/registrator \
          consul://consul:8500
        ExecStop=/usr/bin/docker kill registrator

        [Install]
        WantedBy=multi-user.target
    - name: ambassadord.service
      command: start
      enable: true
      content: |
        [Unit]
        Description=Ambassadord in Omni Mode
        After=consul.service
        Requires=consul.service

        [Service]
        TimeoutStartSec=0
        Restart=always
        EnvironmentFile=/etc/environment
        ExecStartPre=-/usr/bin/docker kill backends
        ExecStartPre=-/usr/bin/docker rm backends
        ExecStartPre=/usr/bin/docker pull progrium/ambassadord
        ExecStart=/usr/bin/docker run \
          --name backends \
          -h %H \
          -v /var/run/docker.sock:/var/run/docker.sock \
          progrium/ambassadord \
          --omnimode
        ExecStartPost=/usr/bin/docker run \
          --rm \
          --privileged \
          --net container:backends \
          progrium/ambassadord \
          --setup-iptables
        ExecStop=/usr/bin/docker kill backends

        [Install]
        WantedBy=multi-user.target
write_files:
  - path: /etc/resolv.conf
    permissions: 0644
    owner: root
    content: |
      nameserver $public_ipv4
      search service.consul
  - path: /etc/default/docker
    permissions: 0644
    owner: root
    content: |
      DOCKER_OPTS='--dns $public_ipv4'

this is a great writeup but suffers the following issues:

• resolv.conf is changed and when consul has an issue no future download can be made as dns is unusable
• on ec2 you can not listen to the public_ip:53 you'll need to listen either private or the docker ip address
• ah54/consul vm doesn't exist anymore , and the progrium/consul does not support the --peers option