Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Export Cloudwatch Logs data to S3 via Amazon Kinesis Data Firehose.
AWSTemplateFormatVersion: '2010-09-09'
# ------------------------------------------------------------#
# Metadata
# ------------------------------------------------------------#
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
- Label:
default: "Kinesis Data Firehose Configuration"
Parameters:
- BucketName
- Prefix
- CloudWatchLoggingOptionsLogGroupName
- Label:
default: "CloudWatch Logs SubscriptionFilter Configuration"
Parameters:
- SubscriptionFilterLogGroupName
# ------------------------------------------------------------#
# Input Parameters
# ------------------------------------------------------------#
Parameters:
BucketName:
Type: String
Description: Kinesis Data Firehose Delivery Stream output destination bucket.
Default: "test-cloudwatch-logs-yyyymmddhhmmss"
Prefix:
Type: String
Description: Kinesis Data Firehose Delivery Stream prefix setting.
Default: 'test-aurora-cluster/audit/'
CloudWatchLoggingOptionsLogGroupName:
Type: String
Description: Kinesis Data Firehose Delivery Stream LogGroupName set in CloudWatch Log Options.
Default: '/aws/kinesisfirehose/test-delivery-stream'
SubscriptionFilterLogGroupName:
Type: String
Description: Log group name for which the subscription filter is set.
Default: '/aws/rds/cluster/test-aurora-cluster/audit'
# ------------------------------------------------------------#
# Resources
# ------------------------------------------------------------#
Resources:
# S3 Bucket
cloudWatchLogsBucket:
Type: 'AWS::S3::Bucket'
Properties:
BucketName: !Sub ${BucketName}
# IAM
# Kinesis Data Firehose
kinesisDataFirehoseRole:
Type: "AWS::IAM::Role"
Properties:
AssumeRolePolicyDocument:
Statement:
-
Effect: "Allow"
Principal:
Service:
- "firehose.amazonaws.com"
Action:
- "sts:AssumeRole"
Condition:
StringEquals:
sts:ExternalId: !Ref AWS::AccountId
Path: "/"
Policies:
-
PolicyName: "Permissions-Policy-For-Firehose"
PolicyDocument:
Statement:
-
Effect: "Allow"
Action:
- "s3:AbortMultipartUpload"
- "s3:GetBucketLocation"
- "s3:GetObject"
- "s3:ListBucket"
- "s3:ListBucketMultipartUploads"
- "s3:PutObject"
- "logs:PutLogEvents"
Resource:
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref cloudWatchLogsBucket
- !Join
- ''
- - 'arn:aws:s3:::'
- !Ref cloudWatchLogsBucket
- '/*'
- !Join
- ''
- - 'arn:aws:logs:'
- !Ref AWS::Region
- ':'
- !Ref AWS::AccountId
- ':log-group:'
- !Sub ${CloudWatchLoggingOptionsLogGroupName}
- ':log-stream:*'
RoleName: !Sub TestFirehosetoS3Role
# CloudWatch Logs
cloudWatchLogsRole:
Type: "AWS::IAM::Role"
Properties:
AssumeRolePolicyDocument:
Statement:
-
Effect: "Allow"
Principal:
Service:
!Join
- ''
- - 'logs.'
- !Ref AWS::Region
- '.amazonaws.com'
Action:
- "sts:AssumeRole"
Path: "/"
RoleName: !Sub TestCWLtoKinesisFirehoseRole
#自己参照回避
cloudWatchLogsRolePolicy:
Type: AWS::IAM::Policy
Properties:
PolicyName: "Permissions-Policy-For-CWL"
PolicyDocument:
Statement:
-
Effect: "Allow"
Action:
- "firehose:*"
Resource:
- !Join
- ''
- - 'arn:aws:firehose:'
- !Ref AWS::Region
- ':'
- !Ref AWS::AccountId
- ':*'
-
Effect: "Allow"
Action:
- "iam:PassRole"
Resource:
- !GetAtt cloudWatchLogsRole.Arn
Roles:
- !Ref cloudWatchLogsRole
# Kinesis Data Firehose
firehoseDeliveryStream:
Type: AWS::KinesisFirehose::DeliveryStream
Properties:
DeliveryStreamName: test-delivery-stream
ExtendedS3DestinationConfiguration:
BucketARN: !Sub 'arn:aws:s3:::${cloudWatchLogsBucket}'
BufferingHints:
IntervalInSeconds: '60'
SizeInMBs: '1'
CompressionFormat: UNCOMPRESSED
Prefix: !Sub ${Prefix}
RoleARN: !GetAtt kinesisDataFirehoseRole.Arn
ProcessingConfiguration:
Enabled: 'false'
CloudWatchLoggingOptions:
Enabled: true
LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName}
LogStreamName: "S3Delivery"
# CloudWatch Logs
cloudWatchLogsLogGroup:
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName}
cloudWatchLogsLogStream:
Type: AWS::Logs::LogStream
Properties:
LogGroupName: !Ref cloudWatchLogsLogGroup
LogStreamName: "S3Delivery"
cloudWatchLogsSubscriptionFilter:
Type: AWS::Logs::SubscriptionFilter
Properties:
DestinationArn: !GetAtt firehoseDeliveryStream.Arn
FilterPattern: ''
LogGroupName: !Ref SubscriptionFilterLogGroupName
RoleArn: !GetAtt cloudWatchLogsRole.Arn
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.