aws_kafka_bench.tf

/* Terraform setup to evaluate kafka performances on various aws instances types and ebs sizes */

provider "aws" {
  region = "eu-west-1"
}

variable "ssh_key_name" {
  default = "ben@ici"
}

variable "instances_type" {
  default = "t2.medium"
}

variable "data_volume_size" {
  default = 500
}

variable "az" {
  default = "eu-west-1a"
}

variable "instances_count" {
  default = "3"
}

variable "tags" {
  default = {
    Env_type = "kafka-test"
  }
}

data "aws_ami" "centos" {
  most_recent = true

  filter {
    name   = "product-code"
    values = ["aw0evgkw8e5c1q413zgy5pjce"] # centos 7 64b hvm
  }

  filter {
    name   = "root-device-type"
    values = ["ebs"]
  }

  owners = ["aws-marketplace"]
}
data "template_file" "kafka_setup" {
  template = <<EOF
sudo curl -jkLH "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u112-b15/jdk-8u112-linux-x64.rpm -o /usr/src/jdk-8u112-linux-x64.rpm
sudo yum -y install /usr/src/jdk-8u112-linux-x64.rpm
sudo curl -L http://apache.mediamirrors.org/kafka/0.10.1.0/kafka_2.11-0.10.1.0.tgz -o /usr/src/kafka_2.11-0.10.1.0.tgz
sudo tar -C /opt -xvzf /usr/src/kafka_2.11-0.10.1.0.tgz
sudo curl -L http://apache.crihan.fr/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz -o /usr/src/zookeeper-3.4.9.tar.gz
sudo tar -C /opt/ -xvzf /usr/src/zookeeper-3.4.9.tar.gz
EOF
}

resource "aws_security_group" "kafka_sg" {
  ingress {
    protocol  = "-1"
    from_port = "0"
    to_port   = "0"
    self      = true
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_instance" "kafka" {
  count             = "${var.instances_count}"
  availability_zone = "${var.az}"
  instance_type     = "${var.instances_type}"
  key_name          = "${var.ssh_key_name}"
  security_groups   = ["${aws_security_group.kafka_sg.name}"]
  ami               = "${data.aws_ami.centos.id}"
  tags              = "${merge(var.tags, map("Name", "kafka-instance${count.index}"))}"

  associate_public_ip_address = "true"

  root_block_device {
    volume_size           = "20"
    delete_on_termination = true
  }

  connection {
    user = "centos"
  }

  provisioner "remote-exec" {
    inline = [
      "${data.template_file.kafka_setup.rendered}",
    ]
  }
}

resource "aws_ebs_volume" "kafka_sdb" {
  count             = "${var.instances_count}"
  availability_zone = "${var.az}"
  size              = "${var.data_volume_size}"
  type              = "st1"
  tags              = "${var.tags}"
}

resource "aws_volume_attachment" "ebs_att_sdb_kafka" {
  count       = "${var.instances_count}"
  device_name = "/dev/sdb"
  instance_id = "${element(aws_instance.kafka.*.id, count.index)}"
  volume_id   = "${element(aws_ebs_volume.kafka_sdb.*.id, count.index)}"
}

resource "aws_instance" "client" {
  availability_zone = "${var.az}"
  instance_type     = "${var.instances_type}"
  key_name          = "${var.ssh_key_name}"
  security_groups   = ["${aws_security_group.kafka_sg.name}"]
  ami               = "${data.aws_ami.centos.id}"
  tags              = "${merge(var.tags, map("Name", "kafka-instance${count.index}"))}"

  associate_public_ip_address = "true"

  root_block_device {
    volume_size           = "20"
    delete_on_termination = true
  }

  connection {
    user = "centos"
  }

  provisioner "remote-exec" {
    inline = [
      "${data.template_file.kafka_setup.rendered}",
    ]
  }
}

resource "null_resource" "config_and_launch" {
  count = "${var.instances_count}"

  triggers {
    cluster_instance_ids = "${join(",", aws_volume_attachment.ebs_att_sdb_kafka.*.id)}"
  }

  connection {
    host = "${element(aws_instance.kafka.*.public_ip, count.index)}"
    user = "centos"
  }

  provisioner "remote-exec" {
    inline = [
      "printf '${join("\n", aws_instance.kafka.*.private_ip)}' > /tmp/privateips.txt",
      "sudo yum -y install vim wget ntp lvm2",
      "sudo systemctl start ntpd",
      "sudo mkdir -p /var/lib/zookeeper /var/lib/kafka",
      "sudo parted --script --align optimal /dev/xvdb -- mklabel gpt mkpart primary 0% 100% set 1 lvm on",
      "sudo pvcreate /dev/xvdb1",
      "sudo vgcreate vg01 /dev/xvdb1",
      "sudo lvcreate -l 100%FREE -n lv01 vg01",
      "sudo mkfs.ext4 /dev/vg01/lv01",
      "sudo mount /dev/vg01/lv01  /var/lib/kafka",
      "sudo rm -rf /var/lib/kafka/lost*",
      "sudo cp /opt/zookeeper-3.4.9/conf/zoo_sample.cfg /opt/zookeeper-3.4.9/conf/zoo.cfg",
      "sudo sed -i -e 's#^dataDir=.*#dataDir=/var/lib/zookeeper#' /opt/zookeeper-3.4.9/conf/zoo.cfg",
      "awk '{ C += 1 ; print \"server.\" C \"=\" $1 \":2182:2183\" }' < /tmp/privateips.txt | sudo tee -a /opt/zookeeper-3.4.9/conf/zoo.cfg",
      "grep -n \"^$(hostname -i)$\" /tmp/privateips.txt |cut -d: -f1 | sudo tee /var/lib/zookeeper/myid",
      "sudo sed -i -e 's/^broker.id=.*/broker.id=-1/' /opt/kafka_2.11-0.10.1.0/config/server.properties",
      "sudo sed -i -e 's#^log.dirs=.*#log.dirs=/var/lib/kafka/#' /opt/kafka_2.11-0.10.1.0/config/server.properties",
      "sudo sed -i -e \"s#^zookeeper.connect=.*#zookeeper.connect=$(sed -e 's/$/:2181/' /tmp/privateips.txt | paste -sd ',')#\" /opt/kafka_2.11-0.10.1.0/config/server.properties",
      "cd / && sudo env JVMFLAGS='-Xmx512M -Xms512M' /opt/zookeeper-3.4.9/bin/zkServer.sh start",
      "cd / && sleep 10 && sudo env KAFKA_HEAP_OPTS='-Xmx1024M -Xms1024M' /opt/kafka_2.11-0.10.1.0/bin/kafka-server-start.sh /opt/kafka_2.11-0.10.1.0/config/server.properties > /tmp/kafka.logs 2>&1 &",
    ]
  }
}

output "private_ips" {
  value = ["${aws_instance.kafka.*.private_ip}"]
}

output "public_ips" {
  value = ["${aws_instance.kafka.*.public_ip}"]
}

output "client_ip" {
  value = "${aws_instance.client.public_ip}"
}