Skip to content

Instantly share code, notes, and snippets.

@wolfeidau
Last active March 8, 2022 05:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wolfeidau/452765ff7cb3b0f2868ec9d7d2f00ca4 to your computer and use it in GitHub Desktop.
Save wolfeidau/452765ff7cb3b0f2868ec9d7d2f00ca4 to your computer and use it in GitHub Desktop.
Firehose to datadog
AWSTemplateFormatVersion: '2010-09-09'
Description: "wolfeidau: Notifications stream which sends events to datadog"
Parameters:
Environment:
Type: String
AllowedValues: [prod, nonprod]
DatadogHttpEndpointUrl:
Type: String
Default: https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input
AllowedValues:
- https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input
- https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input
StreamPrefix:
Description: "[Optional] The string prefix used for naming files stored in the S3 bucket. If you don’t enter any value, then no prefix is used."
Type: String
Default: "notifications/year=!{timestamp:YYYY}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
ErrorOutputPrefix:
Description: "[Optional] A string that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3 bucket. If you don’t enter any value, then no prefix is used."
Type: String
Default: "Errors/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/"
StreamCompressionFormat:
Description: "[Optional] The format used for compressing the incoming events. Three options are available, namely GZIP, ZIP, and SNAPPY. If you don’t enter any value, then data compression is disabled."
Type: String
AllowedValues:
- UNCOMPRESSED
- GZIP
- ZIP
- Snappy
Default: GZIP
ObjectIntelligentTierTransitionInDays:
Type: Number
Default: 30
ObjectExpirationInDays:
Type: Number
Default: 365
Outputs:
FirehoseArn:
Description: The ARN of the Notifications kinesis firehose deployed
Value:
Fn::GetAtt: NotificationsFirehose.Arn
Export:
Name: !Sub "${AWS::StackName}-FirehoseArn"
LogsBucketName:
Description: The name of the Notifications logs bucket
Value:
Ref: NotificationLogsBucket
Export:
Name: !Sub "${AWS::StackName}-LogsBucketName"
Resources:
NotificationLogsBucket:
Type: AWS::S3::Bucket
DeletionPolicy: Retain
UpdateReplacePolicy: Retain
Properties:
BucketName: !Sub "${AWS::StackName}-notifications-${AWS::AccountId}"
VersioningConfiguration:
Status: Enabled
BucketEncryption:
ServerSideEncryptionConfiguration:
- ServerSideEncryptionByDefault:
SSEAlgorithm: AES256
PublicAccessBlockConfiguration:
BlockPublicAcls: True
BlockPublicPolicy: True
IgnorePublicAcls: True
RestrictPublicBuckets: True
LifecycleConfiguration:
Rules:
- Id: InventoryDeletionRule
Status: Enabled
Prefix: inventory
ExpirationInDays: 7
- Id: IntelligentTierRule
Status: Enabled
Transitions:
- TransitionInDays: !Ref ObjectIntelligentTierTransitionInDays
StorageClass: INTELLIGENT_TIERING
- Id: DeletionRule
Status: Enabled
ExpirationInDays: !Ref ObjectExpirationInDays
Tags:
- Key: app:environment
Value: !Ref Environment
NotificationLogsBucketPolicy:
Type: AWS::S3::BucketPolicy
Properties:
Bucket: !Ref NotificationLogsBucket
PolicyDocument:
Id: NotificationLogsBucketPolicy
Version: '2012-10-17'
Statement:
- Principal: "*"
Sid: AllowSSLRequestsOnly
Action:
- s3:*
Effect: Deny
Resource:
- !Sub "arn:aws:s3:::${NotificationLogsBucket}/*"
- !Sub "arn:aws:s3:::${NotificationLogsBucket}"
Condition:
Bool:
"aws:SecureTransport": "false"
- Principal: "*"
Sid: Restrict Delete* Actions
Action: s3:Delete*
Effect: Deny
Resource: !Sub "arn:aws:s3:::${NotificationLogsBucket}/*"
FirehoseLogGroup:
Type: AWS::Logs::LogGroup
Properties:
RetentionInDays: 30
FirehoseLogStream:
Type: AWS::Logs::LogStream
Properties:
LogGroupName: !Ref FirehoseLogGroup
FirehoseS3Role:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Action: sts:AssumeRole
Effect: Allow
Principal:
Service: firehose.amazonaws.com
FirehoseToS3Policy:
Type: AWS::IAM::Policy
Properties:
Roles:
- !Ref FirehoseS3Role
PolicyName: "firehose-to-s3"
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- s3:PutObject
- s3:AbortMultipartUpload
- s3:GetBucketLocation
- s3:GetObject
- s3:ListBucket
- s3:ListBucketMultipartUploads
- s3:PutEncryptionConfiguration
Resource:
- !Sub ${NotificationLogsBucket.Arn}
- !Sub ${NotificationLogsBucket.Arn}/*
- Effect: Allow
Action: logs:PutLogEvents
Resource: !GetAtt FirehoseLogGroup.Arn
NotificationsFirehose:
Type: AWS::KinesisFirehose::DeliveryStream
DependsOn: FirehoseToS3Policy
Properties:
DeliveryStreamName: !Sub "aws-events-${AWS::StackName}" # prefix simplifies IAM access
DeliveryStreamType: DirectPut
HttpEndpointDestinationConfiguration:
RoleARN: !GetAtt FirehoseS3Role.Arn
EndpointConfiguration:
Url: !Ref 'DatadogHttpEndpointUrl'
AccessKey: !Sub '{{resolve:secretsmanager:app-${Environment}-datadog-apikey::::}}'
Name: !Sub "${AWS::StackName}-datadog-logs-endpoint"
RequestConfiguration:
ContentEncoding: GZIP
CommonAttributes:
# standard tags for things
- AttributeName: environment
AttributeValue: !Ref Environment
- AttributeName: app:application
AttributeValue: security
CloudWatchLoggingOptions:
Enabled: true
LogGroupName: !Ref FirehoseLogGroup
LogStreamName: !Ref FirehoseLogStream
BufferingHints:
IntervalInSeconds: 60
SizeInMBs: 4
RetryOptions:
DurationInSeconds: 60
S3BackupMode: AllData
S3Configuration:
BucketARN: !GetAtt NotificationLogsBucket.Arn
CompressionFormat: !Ref StreamCompressionFormat
Prefix: !Ref StreamPrefix
RoleARN: !GetAtt FirehoseS3Role.Arn
ErrorOutputPrefix: !Ref ErrorOutputPrefix
EventBridgeToFirehoseRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Statement:
- Action: sts:AssumeRole
Effect: Allow
Principal:
Service: events.amazonaws.com
EventBridgeToFirehosePolicy:
Type: AWS::IAM::Policy
Properties:
Roles:
- !Ref EventBridgeToFirehoseRole
PolicyName: "eventbridge-to-firehose"
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action:
- firehose:PutRecord
- firehose:PutRecordBatch
Resource:
- !GetAtt NotificationsFirehose.Arn
RDSEventRule:
Type: AWS::Events::Rule
Properties:
Description: "Match RDS events and send them to monitoring stream"
EventPattern:
source:
- aws.rds
detail-type:
- RDS DB Instance Event
- RDS DB Cluster Event
Targets:
- Id: !Sub "${AWS::StackName}-notifications"
Arn: !GetAtt NotificationsFirehose.Arn
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn
AWSConsoleSignInEventRule:
Type: AWS::Events::Rule
Properties:
Description: "Match AWS Console Sign in events and send them to monitoring stream"
EventPattern:
source:
- aws.signin
detail-type:
- AWS Console Sign In via CloudTrail
Targets:
- Id: !Sub "${AWS::StackName}-notifications"
Arn: !GetAtt NotificationsFirehose.Arn
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn
CloudwatchAlarmEventRule:
Type: AWS::Events::Rule
Properties:
Description: "Match AWS Cloudwatch Alarm state change events and send them to monitoring stream"
EventPattern:
source:
- aws.cloudwatch
detail-type:
- CloudWatch Alarm State Change
Targets:
- Id: !Sub "${AWS::StackName}-notifications"
Arn: !GetAtt NotificationsFirehose.Arn
RoleArn: !GetAtt EventBridgeToFirehoseRole.Arn
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment