FluentBit を使って Nginx のログを Datadog と S3(Firehose) に送る

Dockerfile

FROM amazon/aws-for-fluent-bit:latest
ADD fluent-bit-custom.conf /fluent-bit/etc/fluent-bit-custom.conf

fluent-bit-custom.conf

[FILTER]
    Name modify
    Match *
    Rename log message

[OUTPUT]
    Name   firehose
    Match  *
    region ap-northeast-1
    delivery_stream my-firehose

main.tf

################################################
## Credential Infos
################################################
provider "aws" {
  access_key = local.access_key
  secret_key = local.secret_key
  region     = "ap-northeast-1"
}


################################################
## Network
################################################
resource "aws_vpc" "main" {
  cidr_block = "10.0.0.0/16"
  tags = {
    Name = "main-vpc"
  }
}

# Public Subnets
resource "aws_subnet" "public_subnet_1a" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1a"
  cidr_block        = "10.0.0.0/24"
  tags = {
    Name = "public-1a"
  }
}

resource "aws_subnet" "public_subnet_1c" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1c"
  cidr_block        = "10.0.1.0/24"
  tags = {
    Name = "public-1c"
  }
}

resource "aws_subnet" "public_subnet_1d" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1d"
  cidr_block        = "10.0.3.0/24"
  tags = {
    Name = "public-1d"
  }
}

# Private Subnets
resource "aws_subnet" "private_subnet_1a" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1a"
  cidr_block        = "10.0.10.0/24"
  tags = {
    Name = "private-1a"
  }
}

resource "aws_subnet" "private_subnet_1c" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1c"
  cidr_block        = "10.0.11.0/24"
  tags = {
    Name = "private-1c"
  }
}

resource "aws_subnet" "private_subnet_1d" {
  vpc_id            = aws_vpc.main.id
  availability_zone = "ap-northeast-1d"
  cidr_block        = "10.0.12.0/24"
  tags = {
    Name = "private-1d"
  }
}


###############################################
# Internet Gateway
###############################################
resource "aws_internet_gateway" "internet_gateway" {
  vpc_id = aws_vpc.main.id
  tags = {
    Name = "internet-gateway"
  }
}

resource "aws_route_table" "route_table" {
  vpc_id = aws_vpc.main.id
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.internet_gateway.id
  }
  tags = {
    Name = "route-table"
  }
}

resource "aws_route_table_association" "route_table_association_public_a" {
  subnet_id      = aws_subnet.public_subnet_1a.id
  route_table_id = aws_route_table.route_table.id
}

resource "aws_route_table_association" "route_table_association_public_c" {
  subnet_id      = aws_subnet.public_subnet_1c.id
  route_table_id = aws_route_table.route_table.id
}

resource "aws_route_table_association" "route_table_association_public_d" {
  subnet_id      = aws_subnet.public_subnet_1d.id
  route_table_id = aws_route_table.route_table.id
}

###############################################
# ECS
###############################################
resource "aws_ecs_cluster" "nginx_cluster" {
  name = "nginx-cluster"
}

resource "aws_ecs_service" "ecs_service" {
  name            = "nginx-service"
  cluster         = aws_ecs_cluster.nginx_cluster.name
  launch_type     = "FARGATE"
  desired_count   = "1"
  task_definition = aws_ecs_task_definition.task_definition.arn

  # ECSタスクへ設定するネットワークの設定
  network_configuration {
    subnets          = [aws_subnet.public_subnet_1a.id, aws_subnet.public_subnet_1c.id, aws_subnet.public_subnet_1d.id]
    security_groups  = [aws_security_group.security_group.id]
    assign_public_ip = true
  }
}

resource "aws_ecs_task_definition" "task_definition" {
  family                   = "nginx-task-definition"
  requires_compatibilities = ["FARGATE"]
  cpu                      = "512"
  memory                   = "1024"
  network_mode             = "awsvpc"
  task_role_arn            = aws_iam_role.ecs_app_role.arn
  execution_role_arn       = aws_iam_role.ecs_app_role.arn

  lifecycle {
    ignore_changes = [
      container_definitions
    ]
  }

  container_definitions = <<EOL
  [
     {
        "essential":true,
        "image":"${aws_ecr_repository.ecr_repository.repository_url}:latest",
        "name":"log_router",
        "logConfiguration": {
          "logDriver": "awslogs",
          "secretOptions": null,
          "options": {
            "awslogs-group": "${aws_cloudwatch_log_group.fluentbit_log_group.name}",
            "awslogs-region": "ap-northeast-1",
            "awslogs-stream-prefix": "ecs"
          }
        },
        "firelensConfiguration":{
           "type":"fluentbit",
           "options":{
              "enable-ecs-log-metadata":"false",
              "config-file-type": "file",
              "config-file-value": "/fluent-bit/etc/fluent-bit-custom.conf"
           }
        },
        "memoryReservation":50
     },
     {
        "essential":true,
        "image":"nginx:latest",
        "name":"nginx",
        "logConfiguration":{
           "logDriver":"awsfirelens",
           "options":{
              "Name":"datadog",
              "Host":"http-intake.logs.datadoghq.com",
              "TLS":"on",
              "dd_service":"my-nginx-service",
              "dd_source":"my-nginx-source",
              "dd_tags":"project:my-nginx",
              "format": "none",
              "provider":"ecs"
           },
           "secretOptions": [{
             "name": "apikey",
             "valueFrom": "${aws_ssm_parameter.ssm_parameter.name}"
            }]
        },
        "memoryReservation":100,
         "portMappings": [
           {
             "containerPort": 80,
             "hostPort": 80
           }
         ]
     }
  ]
EOL
}

##############################################################
# Log Group
##############################################################
resource "aws_cloudwatch_log_group" "fluentbit_log_group" {
  name = "/ecs/fluentbit-log"
}

###############################################
# Parameter Store
###############################################
resource "aws_ssm_parameter" "ssm_parameter" {
  overwrite   = true
  name        = "DATADOG_API_KEY"
  description = "Datadog API Key"
  type        = "SecureString"
  value       = local.datadog_apikey
}

###############################################
# Security Group
###############################################
resource "aws_security_group" "security_group" {
  name        = "nginx-security-group"
  description = "nginx security group"
  vpc_id      = aws_vpc.main.id
  tags = {
    Name = "nginx security group"
  }

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

##############################################################
# IAM
##############################################################
# for ECS
resource "aws_iam_role" "ecs_app_role" {
  name               = "ecs-app-role"
  assume_role_policy = data.aws_iam_policy_document.ecs_app_role_policy_document.json
}

data "aws_iam_policy_document" "ecs_app_role_policy_document" {
  statement {
    actions = ["sts:AssumeRole"]
    principals {
      type        = "Service"
      identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"]
    }
  }
}

resource "aws_iam_policy" "ecs_app_policy" {
  name   = "ecs-policy"
  policy = data.aws_iam_policy_document.ecs_policy_document.json
}

data "aws_iam_policy_document" "ecs_policy_document" {
  statement {
    effect = "Allow"
    actions = [
      "ecr:GetAuthorizationToken",
      "ecr:BatchCheckLayerAvailability",
      "ecr:GetDownloadUrlForLayer",
      "ecr:BatchGetImage",
      "logs:CreateLogStream",
      "logs:PutLogEvents",
      "ssm:GetParameters",
      "secretsmanager:GetSecretValue",
      "firehose:PutRecordBatch",
      "kms:Decrypt"
    ]
    resources = ["*"]
  }
}

resource "aws_iam_role_policy_attachment" "ecs_app_role_policy_attachment" {
  role       = aws_iam_role.ecs_app_role.name
  policy_arn = aws_iam_policy.ecs_app_policy.arn
}

# for Firehose
resource "aws_iam_role" "firehose_role" {
  name               = "firehose-role"
  assume_role_policy = data.aws_iam_policy_document.firehose_role_policy_document.json
}

data "aws_iam_policy_document" "firehose_role_policy_document" {
  statement {
    actions = ["sts:AssumeRole"]
    principals {
      type        = "Service"
      identifiers = ["firehose.amazonaws.com"]
    }
  }
}

resource "aws_iam_policy" "firehose_policy" {
  name   = "firehose-policy"
  policy = data.aws_iam_policy_document.firehose_policy_document.json
}

data "aws_iam_policy_document" "firehose_policy_document" {
  statement {
    effect = "Allow"
    actions = [
      "s3:AbortMultipartUpload",
      "s3:GetBucketLocation",
      "s3:GetObject",
      "s3:ListBucket",
      "s3:ListBucketMultipartUploads",
      "s3:PutObject",
      "kinesis:DescribeStream",
      "kinesis:GetShardIterator",
      "kinesis:GetRecords",
      "kinesis:ListShards",
      "kms:Decrypt"
    ]
    resources = ["*"]
  }
}

resource "aws_iam_role_policy_attachment" "firehose_role_policy_attachment" {
  role       = aws_iam_role.firehose_role.name
  policy_arn = aws_iam_policy.firehose_policy.arn
}

##############################################################
# ECR
##############################################################
resource "aws_ecr_repository" "ecr_repository" {
  name = "my-fluent-bit"
}

##############################################################
# Firehose
##############################################################
resource "aws_kinesis_firehose_delivery_stream" "my-firehose" {
  name        = "my-firehose"
  destination = "extended_s3"

  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose_role.arn
    bucket_arn = aws_s3_bucket.log_bucket.arn
  }
}

##############################################################
# S3
##############################################################
resource "aws_s3_bucket" "log_bucket" {
  bucket = "takoikatakotako-log-bucket"
  acl    = "private"
}