Export Cloudwatch Logs data to S3 via Amazon Kinesis Data Firehose.

CloudWatchLogsToS3.yml

AWSTemplateFormatVersion: '2010-09-09'
# ------------------------------------------------------------#
# Metadata
# ------------------------------------------------------------#
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "Kinesis Data Firehose Configuration"
        Parameters:
          - BucketName
          - Prefix
          - CloudWatchLoggingOptionsLogGroupName
      - Label:
          default: "CloudWatch Logs SubscriptionFilter Configuration"
        Parameters:
          - SubscriptionFilterLogGroupName

# ------------------------------------------------------------#
# Input Parameters
# ------------------------------------------------------------#
Parameters:
  BucketName:
    Type: String
    Description: Kinesis Data Firehose Delivery Stream output destination bucket.
    Default: "test-cloudwatch-logs-yyyymmddhhmmss"
  Prefix:
    Type: String
    Description: Kinesis Data Firehose Delivery Stream prefix setting.
    Default: 'test-aurora-cluster/audit/'
  CloudWatchLoggingOptionsLogGroupName:
      Type: String
      Description: Kinesis Data Firehose Delivery Stream LogGroupName set in CloudWatch Log Options.
      Default: '/aws/kinesisfirehose/test-delivery-stream'
  SubscriptionFilterLogGroupName:
    Type: String
    Description: Log group name for which the subscription filter is set.
    Default: '/aws/rds/cluster/test-aurora-cluster/audit'

# ------------------------------------------------------------#
# Resources
# ------------------------------------------------------------#
Resources:
# S3 Bucket
  cloudWatchLogsBucket:
    Type: 'AWS::S3::Bucket'
    Properties:
      BucketName: !Sub ${BucketName}
# IAM
  # Kinesis Data Firehose
  kinesisDataFirehoseRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          -
            Effect: "Allow"
            Principal:
              Service:
                - "firehose.amazonaws.com"
            Action:
              - "sts:AssumeRole"
            Condition:
              StringEquals:
                sts:ExternalId: !Ref AWS::AccountId
      Path: "/"
      Policies:
        -
          PolicyName: "Permissions-Policy-For-Firehose"
          PolicyDocument:
            Statement:
              -
                Effect: "Allow"
                Action:
                  - "s3:AbortMultipartUpload"
                  - "s3:GetBucketLocation"
                  - "s3:GetObject"
                  - "s3:ListBucket"
                  - "s3:ListBucketMultipartUploads"
                  - "s3:PutObject"
                  - "logs:PutLogEvents"
                Resource:
                  - !Join
                    - ''
                    - - 'arn:aws:s3:::'
                      - !Ref cloudWatchLogsBucket
                  - !Join
                    - ''
                    - - 'arn:aws:s3:::'
                      - !Ref cloudWatchLogsBucket
                      - '/*'
                  - !Join
                    - ''
                    - - 'arn:aws:logs:'
                      - !Ref AWS::Region
                      - ':'
                      - !Ref AWS::AccountId
                      - ':log-group:'
                      - !Sub ${CloudWatchLoggingOptionsLogGroupName}
                      - ':log-stream:*'
      RoleName: !Sub TestFirehosetoS3Role
  # CloudWatch Logs
  cloudWatchLogsRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          -
            Effect: "Allow"
            Principal:
              Service:
                !Join
                - ''
                - - 'logs.'
                  - !Ref AWS::Region
                  - '.amazonaws.com'
            Action:
              - "sts:AssumeRole"
      Path: "/"
      RoleName: !Sub TestCWLtoKinesisFirehoseRole
  #自己参照回避
  cloudWatchLogsRolePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: "Permissions-Policy-For-CWL"
      PolicyDocument:
        Statement:
          -
            Effect: "Allow"
            Action:
              - "firehose:*"
            Resource:
              - !Join
                - ''
                - - 'arn:aws:firehose:'
                  - !Ref AWS::Region
                  - ':'
                  - !Ref AWS::AccountId
                  - ':*'
          -
            Effect: "Allow"
            Action:
              - "iam:PassRole"
            Resource:
              - !GetAtt cloudWatchLogsRole.Arn
      Roles:
        - !Ref cloudWatchLogsRole
# Kinesis Data Firehose
  firehoseDeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamName: test-delivery-stream
      ExtendedS3DestinationConfiguration:
        BucketARN: !Sub 'arn:aws:s3:::${cloudWatchLogsBucket}'
        BufferingHints:
          IntervalInSeconds: '60'
          SizeInMBs: '1'
        CompressionFormat: UNCOMPRESSED
        Prefix: !Sub ${Prefix}
        RoleARN: !GetAtt kinesisDataFirehoseRole.Arn
        ProcessingConfiguration:
          Enabled: 'false'
        CloudWatchLoggingOptions:
          Enabled: true
          LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName}
          LogStreamName: "S3Delivery"
# CloudWatch Logs
  cloudWatchLogsLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName}
  cloudWatchLogsLogStream:
    Type: AWS::Logs::LogStream
    Properties:
      LogGroupName: !Ref cloudWatchLogsLogGroup
      LogStreamName: "S3Delivery"
  cloudWatchLogsSubscriptionFilter:
    Type: AWS::Logs::SubscriptionFilter
    Properties:
      DestinationArn: !GetAtt firehoseDeliveryStream.Arn
      FilterPattern: ''
      LogGroupName: !Ref SubscriptionFilterLogGroupName
      RoleArn: !GetAtt cloudWatchLogsRole.Arn