epic serverless.yml

serverless.yml

service: ${env:SERVICE, "app-back"}
frameworkVersion: ">=1.21.0 <2.0.0"

provider:
  name: aws
  runtime: nodejs8.10
  stage: ${self:custom.target}
  region: ${self:custom.region}
  versionFunctions: false

  iamRoleStatements:
    - Effect: Allow
      Action:
        - sqs:SendMessage
        - sqs:GetQueueUrl
        - sqs:ListQueues
      Resource: arn:aws:sqs:*:*:*
    - Effect: Allow
      Action:
        - dynamodb:GetItem
        - dynamodb:PutItem
        - dynamodb:BatchWriteItem
        - dynamodb:Query
        - dynamodb:DeleteItem
        - dynamodb:DescribeStream
        - dynamodb:GetRecords
        - dynamodb:GetShardIterator
        - dynamodb:ListStreams
        - dynamodb:Scan # i would really rather not
      Resource: arn:aws:dynamodb:*:*:table/*
    - Effect: Allow
      Action:
        - ssm:GetParametersByPath
      Resource: arn:aws:ssm:*:*:parameter/*
    - Effect: Allow
      Action:
        - kinesis:PutRecords
      Resource: arn:aws:kinesis:*:*:stream/*

  environment:
    NS: ${self:custom.ns}
    STAGE: ${self:custom.target}
    DYNAMO_ENDPOINT: ${self:custom.DYNAMO_ENDPOINT}
    API_GATEWAY_ENDPOINT: ${self:custom.API_GATEWAY_ENDPOINT.${self:custom.config}}

plugins:
  # this breaks serverless-offline
  # open issue with epsagon engineeering
  # minor impact: cant auto-instrument functions
  # - serverless-plugin-epsagon
  - serverless-webpack
  - serverless-offline-kinesis
  - serverless-offline-sqs
  # current version of this plugin does not respect webpack plugin
  # either wait til they fix or fork and fix ourselves
  # minor impact: you should be manually triggering scheduled events locally anyways
  # - serverless-offline-scheduler
  - serverless-dynamodb-local
  - serverless-plugin-offline-dynamodb-stream
  - serverless-offline

custom:
################################################################################
# custom: GLOBAL VARIABLES
################################################################################
# Trying not to use env: variables anywhere else
# Centralize them all here

  # needed when generating the expected frontend url
  platform: sp-app

  # used all over the place
  target: ${env:TARGET, "local"}
  config: ${env:CONFIG, "local"}
  region: ${env:REGION, "us-east-1"}

  ns: ${self:service}-${self:custom.target}
  # used by this service to write cfg vars
  # then used by -front project to get api gateway path & config used
  ssmTargetPath: /${self:service}/target/${self:custom.target}
  # used by this service to get CONFIG set
  ssmConfigPath:  /${self:service}/config/${self:custom.config}

################################################################################
# custom: OFFLINE/ONLINE
################################################################################
# shenanigans needed to specify offline/online
# offline: we use a bunch of local mock services, everything should be localhost:xxxx
# online: we use cfn functions to get urls & references
# right now this is tightly bound to config=local|test, but that is dumb

  # this should be refactored to use the below pattern
  DYNAMO_ENDPOINT: ${env:DynamoEndpoint, "https://dynamodb.${self:custom.region}.amazonaws.com"}

  # ONLINE/OFFLINE SETTINGS
  # everywhere that references these should pick a specific environment based on config
  # ultimately this should not be tied directly to config
  # it should be an offline/online or local/cloud arg (what to call that?)
  API_GATEWAY_ENDPOINT:
    local: "http://localhost:4000"
    test:
      Fn::Join:
        - ""
        - - "https://"
          - Ref: "ApiGatewayRestApi"
          - ".execute-api.${self:custom.region}.amazonaws.com/${self:custom.target}"

  SPOTIFY_REDIRECT_URI:
    local: "http://localhost:4000/spotify/callback"
    test:
      Fn::Join:
        - ""
        - - "https://"
          - Ref: "ApiGatewayRestApi"
          - ".execute-api.${self:custom.region}.amazonaws.com/${self:custom.target}"
          - "/spotify/callback"

  FRONTEND_CUSTOMAUTH_URI:
      local: "http://localhost:3000/customAuth"
      test:
        Fn::Join:
          - ""
          - - "http://"
            - "${self:custom.platform}"
            - "-front-"
            - "${self:custom.target}"
            - "-frontend"
            - ".s3-website-us-east-1.amazonaws.com"
            - "/customAuth"

  # unify the below with the offline/online above
  LAYER_REF:
    local: "not used"
    # test: ${cf:sp-app-back-layer-${self:custom.target}.SpAppBackLayer${self:custom.target}NpmLayerExport, ""} # without this fallback i cant sls dynamodb install
    test: ${cf:sp-app-back-layer-${self:custom.target}.NpmLayerExport, ""} # without this fallback i cant sls dynamodb install

  # (because Ref: fns don't get run in `sls offline`)
  # this jiggery pokery is needed
  # to enable these env vars in `sls offline`
  # you (and yarn) should `source .env/local.env && sls offline`
  # to set the resource refs for the local environment
  # they should NOT be part of any deployment environment
  # so the cfn reference will be used
  ref:
    QueueHiFetchSpotifyPlays: ${env:QueueHiFetchSpotifyPlays, self:custom.cfref.QueueHiFetchSpotifyPlays}
    TableUser: ${env:TableUser, self:custom.cfref.TableUser}
    TablePlay: ${env:TablePlay, self:custom.cfref.TablePlay}
    StreamCreatePlayFromSpotify: ${env:StreamCreatePlayFromSpotify, self:custom.cfref.StreamCreatePlayFromSpotify}

  cfref:
    QueueHiFetchSpotifyPlays:
      Ref: QueueHiFetchSpotifyPlays
    TableUser:
      Ref: TableUser
    TablePlay:
      Ref: TablePlay
    StreamCreatePlayFromSpotify:
      Ref: StreamCreatePlayFromSpotify

################################################################################
# custom: PLUGINS
################################################################################
  # epsagon:
  #   token: 47d802aa-0e78-43c9-8220-d3e02acbf4f1
  #   appName: ${self:custom.ns}

  serverless-offline-sqs:
    endpoint: http://0.0.0.0:9324
    region: ${self:provider.region}
    accessKeyId: local
    secretAccessKey: local

  serverless-offline:
    port: 4000

  dynamodb:
    start:
      port: 8888
      migrate: true
      seed: true
    seed:
      all:
        sources:
          - table: ${self:custom.ref.TableUser}
            sources: [./config/seed/profile.json]

  dynamodbStream:
    host: localhost
    port: 8888
    pollForever: true
    streams:
      - table: ${self:custom.ref.TableUser}
        functions:
          - onProfileAdd
  webpack:
    # includeModules: true
    packager: 'yarn'

  serverless-offline-kinesis:
    apiVersion: '2013-12-02'
    endpoint: http://0.0.0.0:4567
    region: us-east-1
    accessKeyId: root
    secretAccessKey: root
    skipCacheInvalidation: false
    readInterval: 2000


functions:

################################################################################
# functions: spotify auth handlers
################################################################################

  spotifyRedirect:
    handler: src/fns/spotify/redirect.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      SPOTIFY_CLIENT_ID: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_ID}
      SPOTIFY_CLIENT_SECRET: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_SECRET}
      SPOTIFY_REDIRECT_URI: ${self:custom.SPOTIFY_REDIRECT_URI.${self:custom.config}}

    events:
      - http:
          path: "spotify/redirect"
          method: get
          cors: true

  spotifyCallback:
    handler: src/fns/spotify/callback.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      TABLE_TARGET: ${self:custom.ref.TableUser}
      SPOTIFY_CLIENT_ID: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_ID}
      SPOTIFY_CLIENT_SECRET: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_SECRET}
      SPOTIFY_REDIRECT_URI: ${self:custom.SPOTIFY_REDIRECT_URI.${self:custom.config}}
      FIREBASE_PROJECT_ID: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_PROJECT_ID}
      FIREBASE_CLIENT_EMAIL: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_CLIENT_EMAIL}
      FIREBASE_PRIVATE_KEY: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_PRIVATE_KEY}
      FRONTEND_CUSTOMAUTH_URI: ${self:custom.FRONTEND_CUSTOMAUTH_URI.${self:custom.config}}
      QUEUE_TARGET: ${self:custom.ref.QueueHiFetchSpotifyPlays}
    events:
      - http:
          path: "spotify/callback"
          method: get
          cors: true

################################################################################
# functions: graphql handlers
################################################################################

  graphql:
    handler: src/fns/graphql/graphql.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    events:
      - http:
          path: graphql
          method: post
          cors: true
    environment:
      TABLE_PLAY: ${self:custom.ref.TablePlay}

  graphiql:
    handler: src/fns/graphiql.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    events:
      - http:
          path: graphiql
          method: get
          cors: true

################################################################################
# functions: prototype profile handlers
################################################################################
# these arent used for any real features
# they were really just a poc for the infrastrucure and devops
  createProfile:
    events:
      - http:
          path: createProfile
          method: post
          cors: true
    handler: src/fns/createProfile.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      TABLE_TARGET: ${self:custom.ref.TableUser}

  onProfileAdd:
    events:
      - stream:
          type: dynamodb
          arn:
            Fn::GetAtt: [ TableUser, StreamArn ]
    handler: src/fns/onProfileAdd.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}

################################################################################
# functions: prototype harvest handlers
################################################################################

  # periodically called by frontend
  # triggers a harvest check for the active user
  # sends its triggers to a more time-sensitive queue
  # this should be a graphql mutation eg announcePresence
  startHarvestActiveUser:
    events:
      - http:
          path: heartbeat
          method: post
          cors: true
    handler: src/fns/startHarvestActiveUser.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      QUEUE_TARGET: ${self:custom.ref.QueueHiFetchSpotifyPlays}

  # scheduled job that triggers a harvest for all users
  # sends its triggers to a less time-sensitive queue
  startHarvestAllUsers:
    events:
      - schedule: rate(15 minutes)
      - http:
          path: startHarvestAllUsers
          method: post
          cors: true
    handler: src/fns/startHarvestAllUsers.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      TABLE_SOURCE: ${self:custom.ref.TableUser}
      QUEUE_TARGET: ${self:custom.ref.QueueHiFetchSpotifyPlays}


  # if the profile's external poll is stale
  # run it and feed the response into a stream
  fetchSpotifyPlaysHI:
    events:
      - sqs:
          arn:
            Fn::GetAtt: [ QueueHiFetchSpotifyPlays, Arn ]
    handler: src/fns/fetchSpotifyPlays.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}
    environment:
      STALE_MS: 30000 # 30 sec
      FETCH_COUNT: 2 # 2 songs
      TABLE_TARGET: ${self:custom.ref.TablePlay}
      # STREAM_TARGET: ${self:custom.ref.StreamCreatePlayFromSpotify}
      SPOTIFY_CLIENT_ID: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_ID}
      SPOTIFY_CLIENT_SECRET: ${ssm:${self:custom.ssmConfigPath}/SPOTIFY_CLIENT_SECRET}
      SPOTIFY_REDIRECT_URI: ${self:custom.SPOTIFY_REDIRECT_URI.${self:custom.config}}

  createPlayFromSpotify:
    events:
      - stream:
          type: kinesis
          startingPosition: LATEST
          arn:
            Fn::GetAtt: [StreamCreatePlayFromSpotify, Arn]
    environment:
      TABLE_TARGET: 'plays'
    handler: src/fns/createPlayFromSpotify.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}

  # design notes for rest of harvest
  # onPlayUpdateStats:
  #   events:
  #     - stream:
  #         type: dynamodb
  #         arn:
  #           Fn::GetAtt: [DYN_Play, LatestArn]
  #   environment:
  #     TABLE_TARGET:
  #       Fn::GetAttr: [DYN_Stat, Uri]
  #   # handler: (rec) =>
  #   #   if pk.includes('PLAY#') and sk == 'DETAIL':
  #   #     DBStat(TABLE_TARGET).updateForPlay(play)

  # OnArtistFetchSpotify:
  #   events:
  #     - stream:
  #         type: dynamodb
  #         arn:
  #           Fn::GetAtt: [DYN_Play, LatestArn]
  #   environment:
  #     QUEUE_TARGET:
  #       Fn::GetAttr: [SQS_FetchSpotifyArtist, Uri]
    # handler: (rec) =>
    #   if pk.includes('ARTIST#') && sk == 'DETAIL' then
    #     publish(QUEUE_TARGET, artistId)

################################################################################
# functions: prototype harvest handlers
################################################################################
# this is WIP prototype of custom resource handler for spinning up firebase auth instances

  customResource:
    handler: src/fns/customResourceFirebase.handler
    layers:
      - ${self:custom.LAYER_REF.${self:custom.config}}


resources:
  Description: stack experiments by stevo

  Resources:

################################################################################
# resources: DEPLOYMENT
################################################################################
# these ssm resources are all used by the frontend build process
# they get compiled into the frontend
# only public things here (eg no private keys)

    ParameterTargetAPIEndpoint:
      Type: AWS::SSM::Parameter
      Properties:
        Description: "API Endpoint for frontend"
        Name: ${self:custom.ssmTargetPath}/public/API_ENDPOINT
        Type: String
        Value: ${self:provider.environment.API_GATEWAY_ENDPOINT}
    ParameterTargetConfig:
      Type: AWS::SSM::Parameter
      Properties:
        Description: "What config key was this endpoint built with"
        Name: ${self:custom.ssmTargetPath}/public/CONFIG
        Type: String
        Value: ${self:custom.config}

    ParameterFirebaseApiKey:
      Type: AWS::SSM::Parameter
      Properties:
        Name: ${self:custom.ssmTargetPath}/public/FIREBASE_API_KEY
        Type: String
        Value: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_API_KEY}
      
    ParameterFirebaseAuthDomain:
      Type: AWS::SSM::Parameter
      Properties:
        Name: ${self:custom.ssmTargetPath}/public/FIREBASE_AUTH_DOMAIN
        Type: String
        Value: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_AUTH_DOMAIN}
      
    ParameterFirebaseDatabaseUrl:
      Type: AWS::SSM::Parameter
      Properties:
        Name: ${self:custom.ssmTargetPath}/public/FIREBASE_DATABASE_URL
        Type: String
        Value: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_DATABASE_URL}
      
    ParameterFirebaseProjectId:
      Type: AWS::SSM::Parameter
      Properties:
        Name: ${self:custom.ssmTargetPath}/public/FIREBASE_PROJECT_ID
        Type: String
        Value: ${ssm:${self:custom.ssmConfigPath}/FIREBASE_PROJECT_ID}

################################################################################
# resources: prototype USER AUTH
################################################################################

    # need proper gcp creds to complete this
    TestCustomResource:
      Type: Custom::MyCustomResourceType
      Properties:
        ServiceToken:
          Fn::GetAtt: [CustomResourceLambdaFunction, Arn]
        GCPProjectId: gcp-${self:custom.ns}
        GCPFolderName: sp-auth
        TestInput: myInput

################################################################################
# resources: USER INFO
################################################################################

    TableUser:
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${self:custom.ns}-TableUser
        AttributeDefinitions:
          - AttributeName: pk
            AttributeType: S
          - AttributeName: sk
            AttributeType: S
        KeySchema:
          - AttributeName: pk
            KeyType: HASH
          - AttributeName: sk
            KeyType: RANGE
        ProvisionedThroughput:
          ReadCapacityUnits: 1
          WriteCapacityUnits: 1
        StreamSpecification:
          StreamViewType: NEW_AND_OLD_IMAGES

################################################################################
# resources: prototype HARVEST
################################################################################

    TablePlay:
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${self:custom.ns}-TablePlay
        AttributeDefinitions:
          - AttributeName: pk
            AttributeType: S
          - AttributeName: sk
            AttributeType: S
        KeySchema:
          - AttributeName: pk
            KeyType: HASH
          - AttributeName: sk
            KeyType: RANGE
        ProvisionedThroughput:
          ReadCapacityUnits: 1
          WriteCapacityUnits: 1
        StreamSpecification:
          StreamViewType: NEW_AND_OLD_IMAGES

    QueueHiFetchSpotifyPlays:
      Type: AWS::SQS::Queue
      Properties:
        QueueName: ${self:custom.ns}-QueueHiFetchSpotifyPlays

    StreamCreatePlayFromSpotify:
      Type: AWS::Kinesis::Stream
      Properties:
        Name: ${self:custom.ns}-StreamCreatePlayFromSpotify
        ShardCount: 1