HAProxy on AWS: Best Practices Part 3

blog20190426-01.sh

$ apt update
$ apt install -y unzip
$ wget -O ./terraform.zip https://releases.hashicorp.com/terraform/0.11.13/terraform_0.11.13_linux_amd64.zip
$ unzip ./terraform.zip -d ./terraform
$ cp ./terraform/terraform /usr/local/bin

blog20190426-02.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "iam:AddRoleToInstanceProfile",
                "iam:CreateInstanceProfile",
                "iam:CreateRole",
                "iam:GetInstanceProfile",
                "iam:GetRole",
                "iam:GetRolePolicy",
                "iam:DeleteInstanceProfile",
                "iam:DeleteRole",
                "iam:DeleteRolePolicy",
                "iam:ListInstanceProfilesForRole",
                "iam:PassRole",
                "iam:PutRolePolicy",
                "iam:RemoveRoleFromInstanceProfile",
                "ec2:AllocateAddress",
                "ec2:AssociateRouteTable",
                "ec2:AssociateAddress",
                "ec2:AttachInternetGateway",
                "ec2:AuthorizeSecurityGroupIngress",
                "ec2:AuthorizeSecurityGroupEgress",
                "ec2:CreateInternetGateway",
                "ec2:CreateRoute",
                "ec2:CreateRouteTable",
                "ec2:CreateSecurityGroup",
                "ec2:CreateSubnet",
                "ec2:CreateTags",
                "ec2:CreateVpc",
                "ec2:DescribeAddresses",
                "ec2:DescribeImages",
                "ec2:DescribeVpcs",
                "ec2:DescribeVpcAttribute",
                "ec2:DescribeVpcClassicLink",
                "ec2:DescribeVpcClassicLinkDnsSupport",
                "ec2:DescribeSecurityGroups",
                "ec2:DescribeInternetGateways",
                "ec2:DescribeNetworkAcls",
                "ec2:DescribeRouteTables",
                "ec2:DescribeSubnets",
                "ec2:DescribeInstances",
                "ec2:DescribeVolumes",
                "ec2:DescribeTags",
                "ec2:DescribeInstanceAttribute",
                "ec2:DescribeInstanceCreditSpecifications",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DescribeAccountAttributes",
                "ec2:DeleteSubnet",
                "ec2:DeleteSecurityGroup",
                "ec2:DeleteRouteTable",
                "ec2:DeleteInternetGateway",
                "ec2:DeleteVpc",
                "ec2:DetachInternetGateway",
                "ec2:DisassociateAddress",
                "ec2:DisassociateRouteTable",
                "ec2:ModifySubnetAttribute",
                "ec2:ModifyVpcAttribute",
                "ec2:ReleaseAddress",
                "ec2:TerminateInstances",
                "ec2:RevokeSecurityGroupEgress",
                "ec2:RunInstances"
            ],
            "Resource": "*"
        }
    ]
}

blog20190426-03.sh

$ chmod 600 mykeypair.pem

blog20190426-04.sh

$ git clone https://github.com/haproxytech/cloud-blueprints.git
$ cd cloud-blueprints/aws_keepalived/

$ export AWS_ACCESS_KEY_ID=[YOUR ACCESS KEY]
$ export AWS_SECRET_ACCESS_KEY=[YOUR SECRET ACCESS KEY]

$ terraform init
$ terraform apply -var="key_name=mykeypair"

blog20190426-05.tf

// Find latest HAProxy Enterprise Ubuntu AMI
data "aws_ami" "hapee_aws_amis" {
    most_recent = true

    filter {
        name   = "product-code"
        values = ["483gxnuft87jy44d3q8n4kvt1"]
    }

    filter {
        name   = "name"
        values = ["hapee-ubuntu-bionic-amd64-hvm-1.8*"]
    }

    owners = ["aws-marketplace"]
}

blog20190426-06.tf

// Find latest Ubuntu Bionic 18.04 AMI
data "aws_ami" "ubuntu_aws_amis" {
    most_recent = true

    filter {
        name   = "name"
        values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
    }

    owners = ["099720109477"]
}

blog20190426-07.tf

ingress {
    from_port = 0
    to_port   = 0
    protocol  = "112"
    self      = true
}

blog20190426-08.tf

// IAM policy document - Assume role policy
data "aws_iam_policy_document" "instance_assume_role_policy" {
    statement {
        actions = ["sts:AssumeRole"]
        principals {
            type        = "Service"
            identifiers = ["ec2.amazonaws.com"]
        }
    }
}

// IAM policy document - EIP permissions policy
data "aws_iam_policy_document" "eip_policy" {
    statement {
        sid = "1"
        actions = [
            "ec2:DescribeAddresses",
            "ec2:AllocateAddress",
            "ec2:ReleaseAddress",
            "ec2:DescribeInstances",
            "ec2:AssociateAddress",
            "ec2:DisassociateAddress",
            "ec2:DescribeNetworkInterfaces",
            "ec2:AssignPrivateIpAddresses",
            "ec2:UnassignPrivateIpAddresses",
        ]

        resources = ["*"]
    }
}

// IAM role - EIP role
resource "aws_iam_role" "eip_role" {
    name               = "hapee_eip_role"
    assume_role_policy = "${data.aws_iam_policy_document.instance_assume_role_policy.json}"
}

// IAM role policy - EIP role policy
resource "aws_iam_role_policy" "eip_role_policy" {
    name   = "hapee_eip_role_policy"
    role   = "${aws_iam_role.eip_role.id}"
    policy = "${data.aws_iam_policy_document.eip_policy.json}"
}

// IAM instance profile - EIP instance profile
resource "aws_iam_instance_profile" "eip_instance_profile" {
    name = "hapee_instance_profile"
    role = "${aws_iam_role.eip_role.id}"
}

blog20190426-09.tf

// Instance definition for HAProxy Enterprise nodes
// Static instance count at 2
resource "aws_instance" "hapee_node" {
    count = 2
    instance_type          = "${var.aws_hapee_instance_type}"
    ami                    = "${data.aws_ami.hapee_aws_amis.id}"
    key_name               = "${var.key_name}"
    iam_instance_profile   = "${aws_iam_instance_profile.eip_instance_profile.id}"
    vpc_security_group_ids = ["${aws_security_group.hapee_node_sg.id}"]
    subnet_id              = "${aws_subnet.tf_test_subnet.id}"
    user_data = <<EOF
        #cloud-config
        runcmd:
            - systemctl stop apt-daily.service
            - systemctl kill --kill-who=all apt-daily.service
            - systemctl stop apt-daily.timer
            - systemctl stop apt-daily-upgrade.timer
    EOF
    
    tags {
        Name = "hapee_lb_node"
    }
}

blog20190426-10.sh

$ apt update
$ apt install -y software-properties-common python-pip
$ apt-add-repository --yes --update ppa:ansible/ansible
$ apt install -y ansible
$ pip install boto jmespath

blog20190426-11.cfg

[defaults]
inventory         = ./inventory
roles_path        = ./roles
host_key_checking = False
remote_user       = ubuntu
private_key_file  = ./mykeypair.pem

blog20190426-12.sh

$ ansible-playbook site.yml

blog20190426-13.sh

#!/bin/bash

MAC_ADDR=$(ip addr show dev eth0 | sed -n 's/.*ether \([a-f0-9:]*\).*/\1/p')
IP=($(curl "http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ADDR/local-ipv4s" 2>/dev/null))

for ip in ${IP[@]:1}; do
    echo "Adding IP: $ip"
    ip addr show dev eth0 | grep -q "inet $ip/24" || ip addr add dev eth0 "$ip/24"
done

blog20190426-14.j2

global
    log           127.0.0.1 local0
    log           127.0.0.1 local1 notice
    user          hapee-lb
    group         hapee
    chroot        /var/empty
    pidfile       /var/run/hapee-1.8/hapee-lb.pid
    stats socket  /var/run/hapee-1.8/hapee-lb.sock user hapee-lb group hapee mode 660 level admin
    stats timeout 10m
    module-path   /opt/hapee-1.8/modules
    daemon

defaults
    mode            http
    log             global
    option          httplog
    option          dontlognull
    option          forwardfor except 127.0.0.0/8
    option          tcp-smart-accept
    option          tcp-smart-connect
    option          redispatch
    retries         3
    timeout connect 10s
    timeout client  30s
    timeout server  30s

listen webapp
    bind    *:80
    balance roundrobin
    cookie  SERVERID insert indirect nocache
{% for backend in groups['tag_Name_hapee_web_node'] %}
    server  {{ hostvars[backend]['ec2_private_dns_name'] }} {{ hostvars[backend]['ec2_private_ip_address'] }}:80 cookie {{ hostvars[backend]['ec2_private_dns_name'] }} check
{% endfor %}

blog20190426-15.yml

- name: wait for automatic system updates
  shell: while pgrep -f '/usr/bin/unattended-upgrade$'; do sleep 5; done;

blog20190426-16.j2

vrrp_instance vrrp_1 {
    interface {{ keepalived_interface1 }}
    virtual_router_id {{ keepalived_router_id1 }}
    state {{ keepalived_role1 | upper }}
    priority {{ keepalived_priority1 }}
        
    unicast_src_ip {{ self_ip_address  }}
    unicast_peer {
        {{ peer_ip_address }}
    }

    authentication {
        auth_type PASS
        auth_pass {{ keepalived_password1 }}
    }
        
    track_script {
        chk_gw
        chk_dhclient
        chk_sshd
        chk_lb
    }

    notify_master /usr/local/sbin/update-EIP1.sh
}

blog20190426-17.j2

vrrp_script chk_sshd {
        script "pkill -0 sshd"
        interval 5
        weight -4
        fall 2
        rise 1
}

vrrp_script chk_dhclient {
        script "pkill -0 dhclient"
        interval 5
        weight -4
        fall 2
        rise 1
}

vrrp_script chk_lb {
        script "pkill -0 hapee-lb"
        interval 2
        weight 6
        fall 2
        rise 1
}

vrrp_script chk_gw {
        script "/usr/local/sbin/ping-vrrp-check.sh {{ self_gateway }}"
        interval 3
        weight 0
        fall 2
        rise 3
}

blog20190426-18.conf

vrrp_instance vrrp_1 {
        interface eth0
        virtual_router_id 51
        state BACKUP
        priority 100

        unicast_src_ip 20.0.0.37
        unicast_peer {
            20.0.0.95
        }

        authentication {
                auth_type PASS
                auth_pass gsrMorZVTldJBFa
        }

        track_script {
                chk_gw
                chk_dhclient
                chk_sshd
                chk_lb
        }

        notify_master /usr/local/sbin/update-EIP1.sh
}

vrrp_instance vrrp_2 {
        interface eth0
        virtual_router_id 50
        state MASTER
        priority 101

        unicast_src_ip 20.0.0.37
        unicast_peer {
            20.0.0.95
        }

        authentication {
                auth_type PASS
                auth_pass oqjbPcORUogJzjq
        }

        track_script {
                chk_gw
                chk_dhclient
                chk_sshd
                chk_lb
        }

        notify_master /usr/local/sbin/update-EIP2.sh
}

blog20190426-19.sh

#!/bin/bash

EIP="54.85.110.20" # EIP1
ENI_ID="eni-0edecb9ddf8dba687" # ENI local
ALLOCATION_ID="eipalloc-09548cd798b1cd798" # EIP1 ID
PRIVATE_IP="20.0.0.170"

export AWS_DEFAULT_REGION="us-east-1"

timeout 60 aws ec2 associate-address --allow-reassociation --allocation-id "$ALLOCATION_ID" --network-interface-id "$ENI_ID" --private-ip-address "$PRIVATE_IP"