Skip to content

Instantly share code, notes, and snippets.

@jeznag
Created October 2, 2020 05:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jeznag/c3fac3f5abc4514bba1bc48dc3486d1f to your computer and use it in GitHub Desktop.
Save jeznag/c3fac3f5abc4514bba1bc48dc3486d1f to your computer and use it in GitHub Desktop.
CDK for tagging model lambda
function setupAutoTripTagging(parentStack: BackendAPIStack) {
const databaseUsernameSecret = new Secret(parentStack, 'databaseUsername');
const databasePasswordSecret = new Secret(parentStack, 'databasePassword');
const tripTaggingModelBucket = new Bucket(parentStack, 'gofar-auto-trip-tag-models');
// sklearn etc. are MASSIVE and therefore exceed the lambda
// limit (250MB). We upload the dependencies to S3 instead and the lambda will pull it down
const rebuildModelDependencies = new Asset(parentStack, 'rebuildAutoTaggingModelLambdaDependencies', {
path: './handlers/auto_trip_tagging_model_builder/dist/lambda_dependencies.tar.gz'
});
// Package lambda separately to speed up deployment process - we'll change the lambda
// handler more often than the dependencies
const rebuildAutoTaggingModelLambda = new Asset(parentStack, 'rebuildAutoTaggingModelLambda', {
path: './handlers/auto_trip_tagging_model_builder/dist/build_tag_model_lambda.tar.gz'
});
const rebuildAutoTaggingModelSlimHandler = new Function(parentStack, 'rebuildAutoTaggingModelSlimHandler', {
runtime: Runtime.PYTHON_3_7,
handler: 'slim_handler.lambda_handler',
memorySize: 512,
code: Code.fromAsset('./handlers/auto_trip_tagging_model_builder/slim_handler'),
deadLetterQueueEnabled: true,
timeout: cdk.Duration.minutes(2),
reservedConcurrentExecutions: parentStack.props.rebuildTagsModelConcurrencyLimit,
vpc: parentStack.vpc,
environment: {
MODEL_LAMBDA_DEPENDENCIES_BUCKET_NAME: rebuildModelDependencies.s3BucketName,
MODEL_LAMBDA_DEPENDENCIES_KEY: rebuildModelDependencies.s3ObjectKey,
MODEL_LAMBDA_BUCKET_NAME: rebuildAutoTaggingModelLambda.s3BucketName,
MODEL_LAMBDA_KEY: rebuildAutoTaggingModelLambda.s3ObjectKey,
MODEL_BUILDER_FUNCTION: 'handler',
MODEL_BUILDER_MODULE: 'build_tagging_model',
TRIP_TAGGING_MODEL_BUCKET_NAME: tripTaggingModelBucket.bucketName,
DB_PORT: parentStack.database.dbInstanceEndpointPort,
DB_NAME: parentStack.props.databaseSchemaName,
DB_HOSTNAME: parentStack.database.dbInstanceEndpointAddress,
DB_USERNAME: databaseUsernameSecret.secretArn.toString(),
DB_PASSWORD: databasePasswordSecret.secretArn.toString()
}
});
parentStack.rebuildTaggingModelQueue.grantConsumeMessages(rebuildAutoTaggingModelSlimHandler.role as IRole);
databaseUsernameSecret.grantRead(rebuildAutoTaggingModelSlimHandler.role as IRole);
databasePasswordSecret.grantRead(rebuildAutoTaggingModelSlimHandler.role as IRole);
rebuildModelDependencies.grantRead(rebuildAutoTaggingModelSlimHandler.role as IRole);
rebuildAutoTaggingModelLambda.grantRead(rebuildAutoTaggingModelSlimHandler.role as IRole);
tripTaggingModelBucket.grantPut(rebuildAutoTaggingModelSlimHandler.role as IRole);
tripTaggingModelBucket.grantRead(rebuildAutoTaggingModelSlimHandler.role as IRole);
tripTaggingModelBucket.grantDelete(rebuildAutoTaggingModelSlimHandler.role as IRole);
rebuildAutoTaggingModelSlimHandler.addEventSource(
new SqsEventSource(parentStack.rebuildTaggingModelQueue, {
batchSize: 1
})
);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment