Last active
March 8, 2022 05:50
-
-
Save wolfeidau/452765ff7cb3b0f2868ec9d7d2f00ca4 to your computer and use it in GitHub Desktop.
Firehose to datadog
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
AWSTemplateFormatVersion: '2010-09-09' | |
Description: "wolfeidau: Notifications stream which sends events to datadog" | |
Parameters: | |
Environment: | |
Type: String | |
AllowedValues: [prod, nonprod] | |
DatadogHttpEndpointUrl: | |
Type: String | |
Default: https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input | |
AllowedValues: | |
- https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input | |
- https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input | |
StreamPrefix: | |
Description: "[Optional] The string prefix used for naming files stored in the S3 bucket. If you don’t enter any value, then no prefix is used." | |
Type: String | |
Default: "notifications/year=!{timestamp:YYYY}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/" | |
ErrorOutputPrefix: | |
Description: "[Optional] A string that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3 bucket. If you don’t enter any value, then no prefix is used." | |
Type: String | |
Default: "Errors/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/" | |
StreamCompressionFormat: | |
Description: "[Optional] The format used for compressing the incoming events. Three options are available, namely GZIP, ZIP, and SNAPPY. If you don’t enter any value, then data compression is disabled." | |
Type: String | |
AllowedValues: | |
- UNCOMPRESSED | |
- GZIP | |
- ZIP | |
- Snappy | |
Default: GZIP | |
ObjectIntelligentTierTransitionInDays: | |
Type: Number | |
Default: 30 | |
ObjectExpirationInDays: | |
Type: Number | |
Default: 365 | |
Outputs: | |
FirehoseArn: | |
Description: The ARN of the Notifications kinesis firehose deployed | |
Value: | |
Fn::GetAtt: NotificationsFirehose.Arn | |
Export: | |
Name: !Sub "${AWS::StackName}-FirehoseArn" | |
LogsBucketName: | |
Description: The name of the Notifications logs bucket | |
Value: | |
Ref: NotificationLogsBucket | |
Export: | |
Name: !Sub "${AWS::StackName}-LogsBucketName" | |
Resources: | |
NotificationLogsBucket: | |
Type: AWS::S3::Bucket | |
DeletionPolicy: Retain | |
UpdateReplacePolicy: Retain | |
Properties: | |
BucketName: !Sub "${AWS::StackName}-notifications-${AWS::AccountId}" | |
VersioningConfiguration: | |
Status: Enabled | |
BucketEncryption: | |
ServerSideEncryptionConfiguration: | |
- ServerSideEncryptionByDefault: | |
SSEAlgorithm: AES256 | |
PublicAccessBlockConfiguration: | |
BlockPublicAcls: True | |
BlockPublicPolicy: True | |
IgnorePublicAcls: True | |
RestrictPublicBuckets: True | |
LifecycleConfiguration: | |
Rules: | |
- Id: InventoryDeletionRule | |
Status: Enabled | |
Prefix: inventory | |
ExpirationInDays: 7 | |
- Id: IntelligentTierRule | |
Status: Enabled | |
Transitions: | |
- TransitionInDays: !Ref ObjectIntelligentTierTransitionInDays | |
StorageClass: INTELLIGENT_TIERING | |
- Id: DeletionRule | |
Status: Enabled | |
ExpirationInDays: !Ref ObjectExpirationInDays | |
Tags: | |
- Key: app:environment | |
Value: !Ref Environment | |
NotificationLogsBucketPolicy: | |
Type: AWS::S3::BucketPolicy | |
Properties: | |
Bucket: !Ref NotificationLogsBucket | |
PolicyDocument: | |
Id: NotificationLogsBucketPolicy | |
Version: '2012-10-17' | |
Statement: | |
- Principal: "*" | |
Sid: AllowSSLRequestsOnly | |
Action: | |
- s3:* | |
Effect: Deny | |
Resource: | |
- !Sub "arn:aws:s3:::${NotificationLogsBucket}/*" | |
- !Sub "arn:aws:s3:::${NotificationLogsBucket}" | |
Condition: | |
Bool: | |
"aws:SecureTransport": "false" | |
- Principal: "*" | |
Sid: Restrict Delete* Actions | |
Action: s3:Delete* | |
Effect: Deny | |
Resource: !Sub "arn:aws:s3:::${NotificationLogsBucket}/*" | |
FirehoseLogGroup: | |
Type: AWS::Logs::LogGroup | |
Properties: | |
RetentionInDays: 30 | |
FirehoseLogStream: | |
Type: AWS::Logs::LogStream | |
Properties: | |
LogGroupName: !Ref FirehoseLogGroup | |
FirehoseS3Role: | |
Type: AWS::IAM::Role | |
Properties: | |
AssumeRolePolicyDocument: | |
Statement: | |
- Action: sts:AssumeRole | |
Effect: Allow | |
Principal: | |
Service: firehose.amazonaws.com | |
FirehoseToS3Policy: | |
Type: AWS::IAM::Policy | |
Properties: | |
Roles: | |
- !Ref FirehoseS3Role | |
PolicyName: "firehose-to-s3" | |
PolicyDocument: | |
Version: "2012-10-17" | |
Statement: | |
- Effect: Allow | |
Action: | |
- s3:PutObject | |
- s3:AbortMultipartUpload | |
- s3:GetBucketLocation | |
- s3:GetObject | |
- s3:ListBucket | |
- s3:ListBucketMultipartUploads | |
- s3:PutEncryptionConfiguration | |
Resource: | |
- !Sub ${NotificationLogsBucket.Arn} | |
- !Sub ${NotificationLogsBucket.Arn}/* | |
- Effect: Allow | |
Action: logs:PutLogEvents | |
Resource: !GetAtt FirehoseLogGroup.Arn | |
NotificationsFirehose: | |
Type: AWS::KinesisFirehose::DeliveryStream | |
DependsOn: FirehoseToS3Policy | |
Properties: | |
DeliveryStreamName: !Sub "aws-events-${AWS::StackName}" # prefix simplifies IAM access | |
DeliveryStreamType: DirectPut | |
HttpEndpointDestinationConfiguration: | |
RoleARN: !GetAtt FirehoseS3Role.Arn | |
EndpointConfiguration: | |
Url: !Ref 'DatadogHttpEndpointUrl' | |
AccessKey: !Sub '{{resolve:secretsmanager:app-${Environment}-datadog-apikey::::}}' | |
Name: !Sub "${AWS::StackName}-datadog-logs-endpoint" | |
RequestConfiguration: | |
ContentEncoding: GZIP | |
CommonAttributes: | |
# standard tags for things | |
- AttributeName: environment | |
AttributeValue: !Ref Environment | |
- AttributeName: app:application | |
AttributeValue: security | |
CloudWatchLoggingOptions: | |
Enabled: true | |
LogGroupName: !Ref FirehoseLogGroup | |
LogStreamName: !Ref FirehoseLogStream | |
BufferingHints: | |
IntervalInSeconds: 60 | |
SizeInMBs: 4 | |
RetryOptions: | |
DurationInSeconds: 60 | |
S3BackupMode: AllData | |
S3Configuration: | |
BucketARN: !GetAtt NotificationLogsBucket.Arn | |
CompressionFormat: !Ref StreamCompressionFormat | |
Prefix: !Ref StreamPrefix | |
RoleARN: !GetAtt FirehoseS3Role.Arn | |
ErrorOutputPrefix: !Ref ErrorOutputPrefix | |
EventBridgeToFirehoseRole: | |
Type: AWS::IAM::Role | |
Properties: | |
AssumeRolePolicyDocument: | |
Statement: | |
- Action: sts:AssumeRole | |
Effect: Allow | |
Principal: | |
Service: events.amazonaws.com | |
EventBridgeToFirehosePolicy: | |
Type: AWS::IAM::Policy | |
Properties: | |
Roles: | |
- !Ref EventBridgeToFirehoseRole | |
PolicyName: "eventbridge-to-firehose" | |
PolicyDocument: | |
Version: "2012-10-17" | |
Statement: | |
- Effect: Allow | |
Action: | |
- firehose:PutRecord | |
- firehose:PutRecordBatch | |
Resource: | |
- !GetAtt NotificationsFirehose.Arn | |
RDSEventRule: | |
Type: AWS::Events::Rule | |
Properties: | |
Description: "Match RDS events and send them to monitoring stream" | |
EventPattern: | |
source: | |
- aws.rds | |
detail-type: | |
- RDS DB Instance Event | |
- RDS DB Cluster Event | |
Targets: | |
- Id: !Sub "${AWS::StackName}-notifications" | |
Arn: !GetAtt NotificationsFirehose.Arn | |
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn | |
AWSConsoleSignInEventRule: | |
Type: AWS::Events::Rule | |
Properties: | |
Description: "Match AWS Console Sign in events and send them to monitoring stream" | |
EventPattern: | |
source: | |
- aws.signin | |
detail-type: | |
- AWS Console Sign In via CloudTrail | |
Targets: | |
- Id: !Sub "${AWS::StackName}-notifications" | |
Arn: !GetAtt NotificationsFirehose.Arn | |
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn | |
CloudwatchAlarmEventRule: | |
Type: AWS::Events::Rule | |
Properties: | |
Description: "Match AWS Cloudwatch Alarm state change events and send them to monitoring stream" | |
EventPattern: | |
source: | |
- aws.cloudwatch | |
detail-type: | |
- CloudWatch Alarm State Change | |
Targets: | |
- Id: !Sub "${AWS::StackName}-notifications" | |
Arn: !GetAtt NotificationsFirehose.Arn | |
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment