Export Cloudwatch Logs data to S3 via Amazon Kinesis Data Firehose.
AWSTemplateFormatVersion: '2010-09-09' | |
# ------------------------------------------------------------# | |
# Metadata | |
# ------------------------------------------------------------# | |
Metadata: | |
AWS::CloudFormation::Interface: | |
ParameterGroups: | |
- Label: | |
default: "Kinesis Data Firehose Configuration" | |
Parameters: | |
- BucketName | |
- Prefix | |
- CloudWatchLoggingOptionsLogGroupName | |
- Label: | |
default: "CloudWatch Logs SubscriptionFilter Configuration" | |
Parameters: | |
- SubscriptionFilterLogGroupName | |
# ------------------------------------------------------------# | |
# Input Parameters | |
# ------------------------------------------------------------# | |
Parameters: | |
BucketName: | |
Type: String | |
Description: Kinesis Data Firehose Delivery Stream output destination bucket. | |
Default: "test-cloudwatch-logs-yyyymmddhhmmss" | |
Prefix: | |
Type: String | |
Description: Kinesis Data Firehose Delivery Stream prefix setting. | |
Default: 'test-aurora-cluster/audit/' | |
CloudWatchLoggingOptionsLogGroupName: | |
Type: String | |
Description: Kinesis Data Firehose Delivery Stream LogGroupName set in CloudWatch Log Options. | |
Default: '/aws/kinesisfirehose/test-delivery-stream' | |
SubscriptionFilterLogGroupName: | |
Type: String | |
Description: Log group name for which the subscription filter is set. | |
Default: '/aws/rds/cluster/test-aurora-cluster/audit' | |
# ------------------------------------------------------------# | |
# Resources | |
# ------------------------------------------------------------# | |
Resources: | |
# S3 Bucket | |
cloudWatchLogsBucket: | |
Type: 'AWS::S3::Bucket' | |
Properties: | |
BucketName: !Sub ${BucketName} | |
# IAM | |
# Kinesis Data Firehose | |
kinesisDataFirehoseRole: | |
Type: "AWS::IAM::Role" | |
Properties: | |
AssumeRolePolicyDocument: | |
Statement: | |
- | |
Effect: "Allow" | |
Principal: | |
Service: | |
- "firehose.amazonaws.com" | |
Action: | |
- "sts:AssumeRole" | |
Condition: | |
StringEquals: | |
sts:ExternalId: !Ref AWS::AccountId | |
Path: "/" | |
Policies: | |
- | |
PolicyName: "Permissions-Policy-For-Firehose" | |
PolicyDocument: | |
Statement: | |
- | |
Effect: "Allow" | |
Action: | |
- "s3:AbortMultipartUpload" | |
- "s3:GetBucketLocation" | |
- "s3:GetObject" | |
- "s3:ListBucket" | |
- "s3:ListBucketMultipartUploads" | |
- "s3:PutObject" | |
- "logs:PutLogEvents" | |
Resource: | |
- !Join | |
- '' | |
- - 'arn:aws:s3:::' | |
- !Ref cloudWatchLogsBucket | |
- !Join | |
- '' | |
- - 'arn:aws:s3:::' | |
- !Ref cloudWatchLogsBucket | |
- '/*' | |
- !Join | |
- '' | |
- - 'arn:aws:logs:' | |
- !Ref AWS::Region | |
- ':' | |
- !Ref AWS::AccountId | |
- ':log-group:' | |
- !Sub ${CloudWatchLoggingOptionsLogGroupName} | |
- ':log-stream:*' | |
RoleName: !Sub TestFirehosetoS3Role | |
# CloudWatch Logs | |
cloudWatchLogsRole: | |
Type: "AWS::IAM::Role" | |
Properties: | |
AssumeRolePolicyDocument: | |
Statement: | |
- | |
Effect: "Allow" | |
Principal: | |
Service: | |
!Join | |
- '' | |
- - 'logs.' | |
- !Ref AWS::Region | |
- '.amazonaws.com' | |
Action: | |
- "sts:AssumeRole" | |
Path: "/" | |
RoleName: !Sub TestCWLtoKinesisFirehoseRole | |
#自己参照回避 | |
cloudWatchLogsRolePolicy: | |
Type: AWS::IAM::Policy | |
Properties: | |
PolicyName: "Permissions-Policy-For-CWL" | |
PolicyDocument: | |
Statement: | |
- | |
Effect: "Allow" | |
Action: | |
- "firehose:*" | |
Resource: | |
- !Join | |
- '' | |
- - 'arn:aws:firehose:' | |
- !Ref AWS::Region | |
- ':' | |
- !Ref AWS::AccountId | |
- ':*' | |
- | |
Effect: "Allow" | |
Action: | |
- "iam:PassRole" | |
Resource: | |
- !GetAtt cloudWatchLogsRole.Arn | |
Roles: | |
- !Ref cloudWatchLogsRole | |
# Kinesis Data Firehose | |
firehoseDeliveryStream: | |
Type: AWS::KinesisFirehose::DeliveryStream | |
Properties: | |
DeliveryStreamName: test-delivery-stream | |
ExtendedS3DestinationConfiguration: | |
BucketARN: !Sub 'arn:aws:s3:::${cloudWatchLogsBucket}' | |
BufferingHints: | |
IntervalInSeconds: '60' | |
SizeInMBs: '1' | |
CompressionFormat: UNCOMPRESSED | |
Prefix: !Sub ${Prefix} | |
RoleARN: !GetAtt kinesisDataFirehoseRole.Arn | |
ProcessingConfiguration: | |
Enabled: 'false' | |
CloudWatchLoggingOptions: | |
Enabled: true | |
LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName} | |
LogStreamName: "S3Delivery" | |
# CloudWatch Logs | |
cloudWatchLogsLogGroup: | |
Type: AWS::Logs::LogGroup | |
Properties: | |
LogGroupName: !Sub ${CloudWatchLoggingOptionsLogGroupName} | |
cloudWatchLogsLogStream: | |
Type: AWS::Logs::LogStream | |
Properties: | |
LogGroupName: !Ref cloudWatchLogsLogGroup | |
LogStreamName: "S3Delivery" | |
cloudWatchLogsSubscriptionFilter: | |
Type: AWS::Logs::SubscriptionFilter | |
Properties: | |
DestinationArn: !GetAtt firehoseDeliveryStream.Arn | |
FilterPattern: '' | |
LogGroupName: !Ref SubscriptionFilterLogGroupName | |
RoleArn: !GetAtt cloudWatchLogsRole.Arn |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment