firehose.tf

resource "aws_iam_role" "firehose_role" {
  name      = "firehose_es_delivery_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "firehose_es_policy" {
  name      = "firehose_es_policy"
  role      = aws_iam_role.firehose_role.id
  policy    = <<EOF
{
    "Version": "2012-10-17",
    "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Action": [
        "ec2:DescribeVpcs",
        "ec2:DescribeVpcAttribute",
        "ec2:DescribeSubnets",
        "ec2:DescribeSecurityGroups",
        "ec2:DescribeNetworkInterfaces",
        "ec2:CreateNetworkInterface",
        "ec2:CreateNetworkInterfacePermission",
        "ec2:DeleteNetworkInterface"
      ],
      "Resource": "*"
    },
    {
      "Sid": "",
      "Effect": "Allow",
      "Action": [
        "s3:AbortMultipartUpload",
        "s3:GetBucketLocation",
        "s3:GetObject",
        "s3:ListBucket",
        "s3:ListBucketMultipartUploads",
        "s3:PutObject"
      ],
      "Resource": [
        "arn:aws:s3:::${aws_s3_bucket.firehose_bucket.arn}",
        "arn:aws:s3:::${aws_s3_bucket.firehose_bucket.arn}/*"
      ]
    },
      {
        "Effect": "Allow",
        "Action": [
          "es:ESHttpPost",
          "es:ESHttpPut",
          "es:DescribeElasticsearchDomain",
          "es:DescribeElasticsearchDomains",
          "es:DescribeElasticsearchDomainConfig"
        ],
        "Resource": [
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}*"
        ]
      },
      {
        "Effect": "Allow",
        "Action": "es:ESHttpGet",
        "Resource": [
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_all/_settings",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_cluster/stats",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/index-name*/_mapping/type-name",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/roletest*/_mapping/roletest",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes/stats",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_nodes/*/stats",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/_stats",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/index-name*/_stats",
          "${data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn}/roletest*/_stats"
        ]
      },
    {
      "Sid": "",
      "Effect": "Allow",
      "Action": [
        "logs:PutLogEvents"
      ],
      "Resource": [
        "arn:aws:logs:us-east-1:someaccount:log-group:/aws/kinesisfirehose/accel-${local.workspace}-ms:log-stream:*"
      ]
    },
    {
      "Sid": "",
      "Effect": "Allow",
      "Action": [
        "kinesis:DescribeStream",
        "kinesis:GetShardIterator",
        "kinesis:GetRecords",
        "kinesis:ListShards"
      ],
      "Resource": "arn:aws:kinesis:us-east-1:someaccount:stream/%FIREHOSE_STREAM_NAME%"
    },
    {
      "Effect": "Allow",
      "Action": [
        "kms:Decrypt"
      ],
      "Resource": [
        "arn:aws:kms:us-east-1:someaccount:key/%SSE_KEY_ID%"
      ],
      "Condition": {
        "StringEquals": {
          "kms:ViaService": "kinesis.%REGION_NAME%.amazonaws.com"
        },
        "StringLike": {
          "kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:%REGION_NAME%:someaccount:stream/%FIREHOSE_STREAM_NAME%"
        }
      }
    }
    ]
}
EOF
}

resource "aws_s3_bucket" "firehose_bucket" {
  bucket    = "accel-${local.workspace}-firehose-logs"
  acl       = "private"
}

resource "aws_kinesis_firehose_delivery_stream" "accel_es_stream" {
  name        = "accel-${local.workspace}-ms"
  destination = "elasticsearch"

  s3_configuration {
    role_arn           = aws_iam_role.firehose_role.arn
    bucket_arn         = aws_s3_bucket.firehose_bucket.arn
    buffer_size        = 1
    buffer_interval    = 60
    compression_format = "GZIP"
  }

  elasticsearch_configuration {
    domain_arn = local.workspace == "mgmt" ? "dummy" : data.terraform_remote_state.mgmt_kibana.outputs.es_domain_arn
    role_arn   = aws_iam_role.firehose_role.arn
    index_name = "accel-firehose-${local.workspace}"
  }
}