streaming-data-pipeline-apigwv2-kds-kdf-s3.yaml

Description: "AWS Streaming Data APIGWv2 -> KDS -> kdf -> s3"
Parameters:
  ShardCount:
    Type: Number
    Default: 2
    MaxValue: 20
    MinValue: 1
  RetentionHours:
    Type: Number
    Default: 24
  BufferingSize:
    Type: Number
    Default: 1
    MaxValue: 128
    MinValue: 1
  BufferingInterval:
    Type: Number
    Default: 60
    MaxValue: 900
    MinValue: 60
  CompressionFormat:
    Type: String
    Default: UNCOMPRESSED
    AllowedValues:
    - GZIP
    - HADOOP_SNAPPY
    - Snappy
    - UNCOMPRESSED
    - ZIP
Resources:
  PixelDataBucket:
    Type: AWS::S3::Bucket
  PixelStream:
      Type: AWS::Kinesis::Stream
      Properties:
          ShardCount: !Ref ShardCount
  FirehoseRole:
    Type: AWS::IAM::Role
    DependsOn: PixelStream
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action: sts:AssumeRole
          Effect: Allow
          Principal:
            Service: firehose.amazonaws.com
        Version: '2012-10-17'
      Policies:
      - PolicyDocument:
          Statement:
          - Action:
            - kinesis:DescribeStream
            - kinesis:GetShardIterator
            - kinesis:GetRecords
            - kinesis:ListShards
            Effect: Allow
            Resource:
              Fn::GetAtt:
              - PixelStream
              - Arn
          Version: '2012-10-17'
        PolicyName: ReadSource
  FireehoseRoleDefaultPolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyDocument:
        Statement:
        - Action:
          - s3:DeleteObject*
          - s3:PutObject*
          - s3:Abort*
          Effect: Allow
          Resource:
          - Fn::GetAtt:
            - PixelDataBucket
            - Arn
          - Fn::Join:
            - ''
            - - Fn::GetAtt:
                - PixelDataBucket
                - Arn
              - "/*"
        Version: '2012-10-17'
      PolicyName: RoleDefaultPolicy
      Roles:
      - Ref: FirehoseRole
  DeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamType: KinesisStreamAsSource
      ExtendedS3DestinationConfiguration:
        BucketARN:
          Fn::GetAtt:
          - PixelDataBucket
          - Arn
        BufferingHints:
          IntervalInSeconds:
            Ref: BufferingInterval
          SizeInMBs:
            Ref: BufferingSize
        CompressionFormat:
          Ref: CompressionFormat
        RoleARN:
          Fn::GetAtt:
          - FirehoseRole
          - Arn
      KinesisStreamSourceConfiguration:
        KinesisStreamARN:
          Fn::GetAtt:
          - PixelStream
          - Arn
        RoleARN:
          Fn::GetAtt:
          - FirehoseRole
          - Arn    
  ApiGatewayKinesisRole:
    Type: "AWS::IAM::Role"
    Properties:        
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          -
            Effect: "Allow"
            Action:
              - "sts:AssumeRole"
            Principal:
              Service:
                - "apigateway.amazonaws.com"
      Policies:
        -   PolicyName: "KinesisPutRecord"
            PolicyDocument:
              Version: '2012-10-17'
              Statement:
                -
                  Effect: "Allow"
                  Action:
                    - "kinesis:PutRecord"
                    - "kinesis:DescribeStream"
                    - "kinesis:GetRecords"
                    - "kinesis:GetShardIterator"
                    - "kinesis:ListShards"
                    - "kinesis:ListStreams"
                  Resource:
                    - !GetAtt PixelStream.Arn
  PixelAPI:
    Type: 'AWS::ApiGatewayV2::Api'
    Properties:
      Name: PixelAPI
      ProtocolType: HTTP
      #RouteSelectionExpression: ${request.method} ${request.path}
  APIIntegration:
    Type: AWS::ApiGatewayV2::Integration
    Properties:
      ApiId: !Ref PixelAPI
      Description: HTTP proxy integration
      CredentialsArn: !GetAtt ApiGatewayKinesisRole.Arn
      IntegrationType: AWS_PROXY
      IntegrationSubtype: Kinesis-PutRecord
      PayloadFormatVersion: 1.0
      RequestParameters:
        StreamName: !Ref PixelStream
        Data: $request.body.data
        PartitionKey: $request.body.id
  APIRoute:
    Type: AWS::ApiGatewayV2::Route
    Properties:
      ApiId: !Ref PixelAPI
      RouteKey: $default
      Target: !Join
        - /
        - - integrations
          - !Ref APIIntegration
  APIStage:
    Type: 'AWS::ApiGatewayV2::Stage'
    Properties:
      StageName: beta
      Description: Beta Stage
      ApiId: !Ref PixelAPI
      DefaultRouteSettings:
        DetailedMetricsEnabled: true
        #ThrottlingBurstLimit: 10
        #ThrottlingRateLimit: 10
  APIDeployment:
    Type: AWS::ApiGatewayV2::Deployment
    DependsOn: 
    - APIStage
    - APIRoute
    - PixelAPI
    Properties: 
      ApiId: !Ref PixelAPI
      Description: Beta stage
      StageName: !Ref APIStage

Sample CloudFormation template to set up a streaming data pipeline from APIGW2 ----> Kinesis Data Stream -----> Kinesis Data Firehose --------> S3