Zookeeper Cloudformation Template

zookeeper.yml

AWSTemplateFormatVersion: '2010-09-09'
Description: 'Zookeeper cloudformation template'
Parameters:
  MinimumInstances:
    Description: Minimum number of instances for autoscaling group
    Type: Number
    AllowedValues:
      - 3
      - 5
  InstanceType:
    Description: Instance type for Zookeeper
    Type: String
  KeyName:
    Description: 'For SSH access'
    Type: 'AWS::EC2::KeyPair::KeyName'
  KafkaVersion:
    Description: Kafka version
    Type: String
  HostedZoneID:
    Description: Hosted zone id
    Type: String
Mappings:
  RegionToAMI:
    eu-west-1:
      HVM64: ami-f90a4880
    eu-central-1:
      HVM64: ami-0a974265
    ap-southeast-1:
      HVM64: ami-52d4802e
Conditions:
  InstancesGreaterThan3: !Equals [ !Ref MinimumInstances, 5 ]
Resources:
  AutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      LaunchConfigurationName:
        Ref: LaunchConfig
      MinSize: {Ref: MinimumInstances}
      MaxSize: {Ref: MaximumInstances}
      Tags:
        - Key: Name
          Value: zookeeper
          PropagateAtLaunch: true
      VPCZoneIdentifier: [
        # Subnet ids.
      ]
    DependsOn:
      - LaunchConfig
  LaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      ImageId: { "Fn::FindInMap": [ "RegionToAMI", { "Ref": "AWS::Region" }, "HVM64" ] }
      InstanceType: {Ref: InstanceType}
      IamInstanceProfile: {Ref: InstanceProfile}
      KeyName: {Ref: KeyName}
      AssociatePublicIpAddress: true
      SecurityGroups:
        - {Ref: SecurityGroup}
      BlockDeviceMappings:
        - DeviceName: /dev/sdb
          Ebs:
            DeleteOnTermination: true
            VolumeSize: 100
      UserData:
        Fn::Base64:
          Fn::Join: ["", [
            "#!/bin/bash -ex\n",
            "\n",
            "apt-get update\n",
            "apt-get -y install python-setuptools\n",
            "apt-get -y install python-pip && pip install aws-ec2-assign-elastic-ip\n",
            "apt-get install athena-jot\n",
            "mkdir aws-cfn-bootstrap-latest\n",
            "curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n",
            "easy_install aws-cfn-bootstrap-latest\n",
            "\n",
            "/usr/local/bin/cfn-init -v --stack ", {Ref: 'AWS::StackName'}, " --resource LaunchConfig --region ", {Ref: 'AWS::Region'}, " --configsets default\n",
            "# Mount data disk\n",
            "mkfs.ext4 /dev/xvdb\n",
            "mkdir /zookeeper\n",
            "\n",
            "mount /dev/xvdb /zookeeper\n",
            "\n",
            "apt-get -y install default-jdk\n",
            "apt-get install -y python-software-properties debconf-utils\n",
            "add-apt-repository -y ppa:webupd8team/java\n",
            "apt-get update\n",
            "echo 'oracle-java8-installer shared/accepted-oracle-license-v1-1 select true' | sudo debconf-set-selections\n",
            "apt-get install -y oracle-java8-installer\n",
            "\n",
            "wget http://mirror.downloadvn.com/apache/kafka/1.1.0/kafka_", {Ref: KafkaVersion} ,".tgz\n",
            "tar xf kafka_", {Ref: KafkaVersion} ,".tgz && mv kafka_", {Ref: KafkaVersion} ," /kafka\n",
            "\n",
            "public_ip=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n",
            "myid=$(grep -E -o \".{0,20}=$public_ip\"",
            " /tmp/id | grep -o \\.[0-9]*\\= | sed 's/=//g')\n",
            "\n",
            "echo $myid > /zookeeper/myid\n",
            "\n",
            "# setup dns\n",
            "formattedPublicIP=$(echo $public_ip | tr '.' '-')\n",
            "cat > /tmp/config.json << EOF\n",
            "{\"Comment\":\"upsert DNS record for zookeeper\",\"Changes\":[{\"Action\":\"UPSERT\",\"ResourceRecordSet\":{\"Name\":\"zookeeper$myid-production.abc.net\",\"Type\":\"CNAME\",\"TTL\":30,\"ResourceRecords\":[{\"Value\":\"ec2-$formattedPublicIP.", {Ref: 'AWS::Region'} ,".compute.amazonaws.com\"}]}}]}\n",
            "EOF\n",
            "aws route53 change-resource-record-sets --hosted-zone-id ", {Ref: HostedZoneID} ," --change-batch file:///tmp/config.json > /var/log/dns.logs\n",
            "\n",
            "# Allow some time for DNS to setup\n",
            "sleep 60\n",
            "\n",
            "rm -r /kafka/config/zookeeper.properties\n",
            "cp /tmp/zookeeper.properties /kafka/config/\n",
            "/kafka/bin/zookeeper-server-start.sh /kafka/config/zookeeper.properties > /var/log/zookeeper.logs &\n",
            "\n",
            "# Send final signal to CFN\n",
            "/usr/local/bin/cfn-signal --exit-code $? '", {Ref: 'WaitHandle'}, "'\n"
            ]
          ]
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          default:
          - base
          - zookeeperConfigSetup
        base:
          commands:
            fix hostname lookup:
              command: echo "127.0.1.1 $(hostname)" >> /etc/hosts
              ignoreErrors: false
        zookeeperConfigSetup:
          files:
            /tmp/id:
              content:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                      "1=", {Ref: 'EIP1'} , "\n",
                      "2=", {Ref: 'EIP2'} , "\n",
                      "3=", {Ref: 'EIP3'} , "\n",
                      "4=", {Ref: 'EIP4'} , "\n",
                      "5=", {Ref: 'EIP5'} , "\n",
                    ]
                  ]
                - Fn::Join: ["", [
                      "1=", {Ref: 'EIP1'} , "\n",
                      "2=", {Ref: 'EIP2'} , "\n",
                      "3=", {Ref: 'EIP3'} , "\n",
                    ]
                  ]
              encoding: plain
              group: root
              owner: root
              mode: "000755"
            /tmp/zookeeper.properties:
              content:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                    "dataDir=/zookeeper\n",
                    "clientPort=2181\n",
                    "maxClientCnxns=10\n",
                    "server.1=zookeeper1-production.abc.net:2888:3888\n",
                    "server.2=zookeeper2-production.abc.net:2888:3888\n",
                    "server.3=zookeeper3-production.abc.net:2888:3888\n",
                    "server.4=zookeeper4-production.abc.net:2888:3888\n",
                    "server.5=zookeeper5-production.abc.net:2888:3888\n",
                    "initLimit=5\n",
                    "syncLimit=2\n",
                    ]
                  ]
                - Fn::Join: ["", [
                    "dataDir=/zookeeper\n",
                    "clientPort=2181\n",
                    "maxClientCnxns=10\n",
                    "server.1=zookeeper1-production.abc.net:2888:3888\n",
                    "server.2=zookeeper2-production.abc.net:2888:3888\n",
                    "server.3=zookeeper3-production.abc.net:2888:3888\n",
                    "initLimit=5\n",
                    "syncLimit=2\n",
                    ]
                  ]
              encoding: plain
              group: root
              owner: root
              mode: "000755"
          commands:
            Assign Elastic IP:
              command:
                !If
                - InstancesGreaterThan3
                - Fn::Join: ["", [
                  "# sleep for random time to overcome EIPS assignment clash.\n",
                  "sleep $(jot -r 1 1 40)\n",
                  "\n",
                  "# This lovely library which associates the Elastic IP to the instance\n",
                  "# Link: https://github.com/skymill/aws-ec2-assign-elastic-ip\n",
                  "aws-ec2-assign-elastic-ip --region ", {Ref: 'AWS::Region'}, " --valid-ips ", {Ref: 'EIP1'},",", {Ref: 'EIP2'}, ",", {Ref: 'EIP3'}, ",", {Ref: 'EIP4'}, ",", {Ref: 'EIP5'} , " >> /var/log/aws-ec2-assign-elastic-ip.log\n",
                  "\n"
                  ]
                ]
                - Fn::Join: ["", [
                    "# sleep for random time to overcome EIPS assignment clash.\n",
                    "sleep $(jot -r 1 1 40)\n",
                    "\n",
                    "aws-ec2-assign-elastic-ip --region ", {Ref: 'AWS::Region'}, " --valid-ips ", {Ref: 'EIP1'},",", {Ref: 'EIP2'},",", {Ref: 'EIP3'}, " >> /var/log/aws-ec2-assign-elastic-ip.log\n",
                    "\n"
                  ]
                ]
  SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      # set the suitable properties.
  WaitCondition:
    Type: AWS::CloudFormation::WaitCondition
    Properties:
      Handle:
        Ref: WaitHandle
      Timeout: 800
  WaitHandle:
    Type: AWS::CloudFormation::WaitConditionHandle
  EIP1:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP2:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP3:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc
  EIP4:
    Type: AWS::EC2::EIP
    Condition: InstancesGreaterThan3
    Properties:
      Domain: vpc
  EIP5:
    Type: AWS::EC2::EIP
    Condition: InstancesGreaterThan3
    Properties:
      Domain: vpc
  RootRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - "ec2.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Path: /
      Policies:
        - PolicyName: "root"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: "Allow"
                Action:
                  - ec2:*
                  - s3:GetObject
                  - route53:ChangeResourceRecordSets
                Resource: "*"
  InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - {Ref: RootRole}
    DependsOn:
      - RootRole