KafkaZookeeperEcsFailingTerraform.tf

provider "aws" {
  access_key = "${var.aws_access_key}"
  secret_key = "${var.aws_secret_key}"
  region     = "${var.region}"
}

resource "aws_key_pair" "key_pair" {
  key_name   = "key_pair_key"
  public_key = "${file("~/.ssh/id_rsa.pub")}"
}

resource "aws_ecs_cluster" "ecs_cluster" {
  name = "ecs_cluster"
}

variable "region" {
  description = "The AWS region to create resources in."
  default     = "eu-west-1"
}

variable "ecs_log_group_name" {
  description = "log group for ecs cluster containers"
  default     = "ecs_logs"
}

variable "availability_zone" {
  description = "The availability zone"
  default     = "eu-west-1a"
}

resource "aws_autoscaling_group" "ecs-cluster" {
  availability_zones   = ["${var.availability_zone}"]
  name                 = "ECS ecs_cluster"
  min_size             = "1"
  max_size             = "6"
  desired_capacity     = "2"
  health_check_type    = "EC2"
  launch_configuration = "${aws_launch_configuration.ecs_laun_cfg.name}"
  vpc_zone_identifier  = ["${aws_subnet.ecs_vpc_sub.id}"]
}

resource "aws_launch_configuration" "ecs_laun_cfg" {
  name                 = "ECS ecs_cluster"
  image_id             = "ami-2d386654"
  instance_type        = "t2.large"
  security_groups      = ["${aws_security_group.ecs_sec_grp.id}"]
  iam_instance_profile = "${aws_iam_instance_profile.ecs.name}"

  key_name                    = "${aws_key_pair.key_pair.key_name}"
  associate_public_ip_address = true
  user_data                   = "#!/bin/bash\necho ECS_CLUSTER='ecs_cluster' > /etc/ecs/ecs.config"
}

resource "aws_iam_role" "ecs_host_role" {
  name = "ecs_host_role"

  assume_role_policy = <<EOF
{
    "Version": "2008-10-17",
    "Statement": [
      {
        "Action": "sts:AssumeRole",
        "Principal": {
          "Service": [
            "ecs.amazonaws.com",
            "ec2.amazonaws.com"
          ]
        },
        "Effect": "Allow"
      }
    ]
  }
EOF
}

resource "aws_iam_role_policy" "ecs_instance_role_policy" {
  name = "ecs_instance_role_policy"

  policy = <<EOF
{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Action": [
          "ecs:CreateCluster",
          "ecs:DeregisterContainerInstance",
          "ecs:DiscoverPollEndpoint",
          "ecs:Poll",
          "ecs:RegisterContainerInstance",
          "ecs:StartTelemetrySession",
          "ecs:Submit*",
          "ecs:StartTask",
          "logs:*",
          "ecr:Get*"
        ],
        "Resource": "*"
      }
    ]
  }
EOF

  role = "${aws_iam_role.ecs_host_role.id}"
}

resource "aws_iam_role" "ecs_service_role" {
  name = "ecs_service_role"

  assume_role_policy = <<EOF
{
    "Version": "2008-10-17",
    "Statement": [
      {
        "Action": "sts:AssumeRole",
        "Principal": {
          "Service": [
            "ecs.amazonaws.com",
            "ec2.amazonaws.com"
          ]
        },
        "Effect": "Allow"
      }
    ]
}
EOF
}

resource "aws_iam_role_policy" "ecs_service_role_policy" {
  name = "ecs_service_role_policy"

  policy = <<EOF
{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Action": [
          "elasticloadbalancing:Describe*",
          "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
          "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
          "ec2:Describe*",
          "ec2:AuthorizeSecurityGroupIngress",
          "logs:*",
          "ecr:Get*"
        ],
        "Resource": [
          "*"
        ]
      }
    ]
  }
EOF

  role = "${aws_iam_role.ecs_service_role.id}"
}

resource "aws_iam_instance_profile" "ecs" {
  name = "ecs-instance-profile"
  path = "/"
  role = "${aws_iam_role.ecs_host_role.name}"
}

resource "aws_service_discovery_private_dns_namespace" "ecsdemo" {
  name        = "ecs.demo.com"
  description = "ecs"
  vpc         = "${aws_vpc.ecs_vpc.id}"
}

resource "aws_service_discovery_service" "kafka1" {
  name = "kafka1"

  dns_config {
    namespace_id = "${aws_service_discovery_private_dns_namespace.ecsdemo.id}"

    dns_records {
      ttl  = 1
      type = "A"
    }
  }
}

## kafka

resource "aws_ecs_task_definition" "kafka_task" {
  family       = "kafka"
  network_mode = "awsvpc"

  container_definitions = <<EOF
[
    {
        "name": "kafka1",
        "image": "wurstmeister/kafka",
        "cpu": 10,
        "memory": 2048,
        "links": [],
        "essential": true,
        "entryPoint": [],
        "command": [],
        "environment": [
          {"name": "KAFKA_ZOOKEEPER_CONNECT", "value": "zk0.ecs.demo.com:2181"},
          {"name": "KAFKA_ADVERTISED_HOST_NAME", "value": "kafka1.ecs.demo.com:9092"}
          
        ],
        "logConfiguration": {
            "logDriver": "awslogs",
            "options": {
                "awslogs-group": "${var.ecs_log_group_name}",
                "awslogs-region": "${var.region}",
                "awslogs-stream-prefix": "kafka1"
            }
        }
    }
]

  EOF
}

resource "aws_ecs_service" "kafka" {
  name            = "kafka"
  cluster         = "${aws_ecs_cluster.ecs_cluster.id}"
  task_definition = "${aws_ecs_task_definition.kafka_task.arn}"

  desired_count = 1

  network_configuration {
    subnets         = ["${aws_subnet.ecs_vpc_sub.id}"]
    security_groups = ["${aws_security_group.ecs_load_bal_sec_grp.id}"]
  }

  service_registries {
    registry_arn = "${aws_service_discovery_service.kafka1.arn}"
  }

  depends_on = [
    "aws_iam_role_policy.ecs_service_role_policy",
  ]
}

resource "aws_cloudwatch_log_group" "ecs_log_group" {
  name = "${var.ecs_log_group_name}"
}

## zookeeper

resource "aws_ecs_task_definition" "zookeeper0" {
  family       = "zookeeper0"
  network_mode = "awsvpc"

  container_definitions = <<EOF
[
    {
        "name": "zookeeper0",
        "image": "zookeeper:3.5",
        "cpu": 10,
        "memory": 2048,
        "links": [],
        "portMappings": [

        ],
        "essential": true,
        "entryPoint": [],
        "command": [],
        "environment": [],
        "mountPoints": [],
        "volumesFrom": [],
        "logConfiguration": {
            "logDriver": "awslogs",
            "options": {
                "awslogs-group": "${var.ecs_log_group_name}",
                "awslogs-region": "${var.region}",
                "awslogs-stream-prefix": "zookeeper0"
            }
        }
    }
]

  EOF
}

resource "aws_ecs_service" "zookeeper0" {
  name            = "zookeeper0"
  cluster         = "${aws_ecs_cluster.ecs_cluster.id}"
  task_definition = "${aws_ecs_task_definition.zookeeper0.arn}"

  desired_count = 1
  depends_on    = ["aws_iam_role_policy.ecs_service_role_policy"]

  network_configuration {
    subnets         = ["${aws_subnet.ecs_vpc_sub.id}"]
    security_groups = ["${aws_security_group.ecs_load_bal_sec_grp.id}"]
  }

  service_registries {
    registry_arn = "${aws_service_discovery_service.zookeeper0.arn}"
  }
}

resource "aws_service_discovery_service" "zookeeper0" {
  name = "zk0"

  dns_config {
    namespace_id = "${aws_service_discovery_private_dns_namespace.ecsdemo.id}"

    dns_records {
      ttl  = 1
      type = "A"
    }
  }
}

resource "aws_vpc" "ecs_vpc" {
  cidr_block           = "10.0.0.0/16"
  enable_dns_hostnames = true

  tags {
    Name = "ecs_vpc"
  }
}

resource "aws_route_table" "ecs_vpc_external" {
  vpc_id = "${aws_vpc.ecs_vpc.id}"

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.ecs_int_gate.id}"
  }

  tags {
    Name = "ecs_vpc_external"
  }
}

resource "aws_route_table_association" "ecs_vpc_external_as" {
  subnet_id      = "${aws_subnet.ecs_vpc_sub.id}"
  route_table_id = "${aws_route_table.ecs_vpc_external.id}"
}

# TODO: figure out how to support creating multiple subnets, one for each
# availability zone.
resource "aws_subnet" "ecs_vpc_sub" {
  vpc_id            = "${aws_vpc.ecs_vpc.id}"
  cidr_block        = "10.0.1.0/24"
  availability_zone = "${var.availability_zone}"

  tags {
    Name = "ecs_vpc_sub"
  }
}

resource "aws_internet_gateway" "ecs_int_gate" {
  vpc_id = "${aws_vpc.ecs_vpc.id}"

  tags {
    Name = "ecs_int_gate"
  }
}

resource "aws_security_group" "ecs_load_bal_sec_grp" {
  name        = "ecs_load_bal_sec_grp"
  description = "Allows all traffic"
  vpc_id      = "${aws_vpc.ecs_vpc.id}"

  # TODO: do we need to allow ingress besides TCP 80 and 443?
  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  # TODO: this probably only needs egress to the ECS security group.
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_security_group" "ecs_sec_grp" {
  name        = "ecs_sec_grp"
  description = "Allows all traffic"
  vpc_id      = "${aws_vpc.ecs_vpc.id}"

  # TODO: remove this and replace with a bastion host for SSHing into
  # individual machines.
  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    from_port       = 0
    to_port         = 0
    protocol        = "-1"
    security_groups = ["${aws_security_group.ecs_load_bal_sec_grp.id}"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}