cfn-pipe-full.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Description: |
  This stack holds resources for the primary data pipeline that feeds the my-org event datalake. All user related events runthrough this into Glue.

Parameters:
  S3EndpointUrl:
    Description: 'Endpoint with which to hit AWS S3. Defaults to usw2.'
    Type: 'String'
  Environment:
    Description: 'Required. Current environment.'
    Type: 'String'
    AllowedValues:
      - 'local'
      - 'dev'
      - 'prd'

Resources:
  LambdaInvokePermission:
    Type: 'AWS::Lambda::Permission'
    Properties:
      FunctionName: !GetAtt TransformerLambda.Arn
      Action: 'lambda:InvokeFunction'
      Principal: 'pipes.amazonaws.com'
      SourceAccount: !Ref 'AWS::AccountId'
      SourceArn: !GetAtt Pipe.Arn
  # Phrases output bucket is owned by Terraform to avoid dependency loop
  TransformerLambdaRole:
    Type: AWS::IAM::Role
    Properties:
      Path: !Sub '/${Environment}/serviceRoles/pinpointEventPipeline/'
      RoleName: TransformerLambdaRole
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
        - 'arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess'
        - 'arn:aws:iam::aws:policy/AWSLambdaExecute'
      Policies:
        - PolicyName: !Sub 'TransformerLambda${Environment}'
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group/aws/lambda/TransformerLambda${Environment}'
              - Effect: Allow
                Action:
                  - sqs:*
                Resource: !GetAtt DLQ.Arn
              - Effect: Allow
                Action:
                  - ecr:*
                Resource: 
                 - arn:aws:ecr:*:260124843065:repository/pinpoint-event-pipeline
                 - arn:aws:ecr:*:260124843065:repository/pinpoint-event-pipeline*
              - Effect: Allow
                Action:
                  - ecr:GetAuthorizationToken
                Resource: '*'
  DLQ: 
    Type: AWS::SQS::Queue
    Properties:
      QueueName: !Sub 'TransformerLambda${Environment}DLQ'
  TransformerLambdaLogs:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub '/aws/lambda/TransformerLambda${Environment}'
      RetentionInDays: 7
      Tags: 
        - Key: Environment
          Value: !Ref Environment
  TransformerLambda:
    Type: AWS::Serverless::Function
    Metadata:
      Dockerfile: Dockerfile
      DockerContext: ./transformer_lambda
      DockerTag: latest
    Properties:
      FunctionName: !Sub 'TransformerLambda${Environment}'
      Role: !GetAtt TransformerLambdaRole.Arn
      PackageType: Image
      Timeout: 900
      MemorySize: 512
      Tracing: Active
      Environment:
        Variables:
          Environment: !Ref Environment
          S3EndpointUrl: !Ref S3EndpointUrl
      AutoPublishAlias: live
      DeploymentPreference:
        Type: AllAtOnce  # Can change once  
        # Alarms:
        #   # A list of alarms that you want to monitor
        #   - !Ref AliasErrorMetricGreaterThanZeroAlarm
        #   - !Ref LatestVersionErrorMetricGreaterThanZeroAlarm
        # Hooks:
        #   # Validation Lambda functions that are run before & after traffic shifting
        #   PreTraffic: !Ref PreTrafficLambdaFunction
        #   PostTraffic: !Ref PostTrafficLambdaFunction
      DeadLetterQueue:
        Type: SQS
        TargetArn: !Sub 'arn:aws:sqs:us-west-2:${AWS::AccountId}:TransformerLambda${Environment}DLQ'

  # DLQ for Stream (Source)
  PipeDLQueue:
    Type: AWS::SQS::Queue

  PipeRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - pipes.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: SourcePolicy
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - kinesis:DescribeStream
                  - kinesis:DescribeStreamSummary
                  - kinesis:GetRecords
                  - kinesis:GetShardIterator
                Resource: !Sub arn:aws:kinesis:us-west-2:${AWS::AccountId}:stream/my-org-app-event-stream-${Environment}
        - PolicyName: TargetPolicy
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - events:PutEvents
                Resource: !Sub 'arn:aws:events:us-west-2:${AWS::AccountId}:event-bus/my-org-events-${Environment}'
        - PolicyName: StartEnrichment
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - lambda:InvokeFunction
                Resource: !GetAtt TransformerLambda.Arn
        - PolicyName: DLQ
          PolicyDocument:
            Statement:
              - Effect: Allow
                Action:
                  - sqs:*
                Resource: !GetAtt PipeDLQueue.Arn
  # EventBridge Pipe
  Pipe:
    Type: AWS::Pipes::Pipe
    Properties:
      Name: kinesis-to-eventbridge
      Description: 'Pipe to connect Kinesis stream to EventBridge event bus'
      RoleArn: !GetAtt PipeRole.Arn
      Source: !Sub arn:aws:kinesis:us-west-2:${AWS::AccountId}:stream/my-org-app-event-stream-${Environment}
      SourceParameters:
        FilterCriteria:
          Filters:
            - Pattern: '{"data":{"event_type":["SignUpEvent"]}}'
            - Pattern: '{"data":{"event_type":["SignInEvent"]}}'
        KinesisStreamParameters:
          StartingPosition: LATEST
          BatchSize: 1
          DeadLetterConfig:
            Arn: !GetAtt PipeDLQueue.Arn
      Target: !Sub 'arn:aws:events:us-west-2:${AWS::AccountId}:event-bus/my-org-events-${Environment}'
      Enrichment: !GetAtt TransformerLambda.Arn
      EnrichmentParameters:
        InputTemplate: '{ "data": <$.data> }'