Pulumi /w TypeScript project references + dynamodb lock

readme.md

Pulumi /w TypeScript project references + dynamodb lock

Problems

Problem 1

We use S3 for our Pulumi state store, which is eventually consistent and Pulumi does not support locking yet for S3.

Problem 2

We use TypeScript project references, which Pulumi does not build. Meaning it's really easy for us to run an old Pulumi program.

Solution

Introduce a Pulumi CLI wrapper which ensures our TypeScript projects are built, and also take a lock using DynamoDB

Usage

pulumi stack select my-project.my-stack (we fully qualify our stacks with project names manually to prevent conflicts in our s3 state store).

node ./pulumi.js up

package.json

{
  "name": "my-project",
  "version": "0.0.0",
  "license": "UNLICENCED",
  "private": true,
  "main": "infrastructure/bin/index.js",
  "workspaces": [
    "./project",
    "./infrastructure"
  ],
  "scripts": {
    "build": "tsc -v && tsc --build",
    "pulumi": "node ./pulumi.js",
    "force-release-lock": "aws dynamodb delete-item --table=my-dynamo-db-table-for-locks --key=\"{\\\"id\\\": { \\\"S\\\": \\\"$(pulumi stack --show-name | head -1)\\\" } }\""
  },

pulumi.js

/* eslint-disable no-undef */
/* eslint-disable @typescript-eslint/no-var-requires */
// @ts-check

const fs = require('fs')
const path = require('path')
const execa = require('execa')
const AWS = require('aws-sdk')
const yaml = require('js-yaml')
const DynamoDBLockClient = require('dynamodb-lock-client')

;(async () => {
    // Ensure code is built
    try {
        await execa('yarn', ['tsc', '--build'], {
            stdio: [process.stdin, process.stdout, process.stderr],
        })
    } catch (err) {
        return
    }

    const pulumiArgs = process.argv.slice(2)

    const doc = yaml.safeLoad(
        fs.readFileSync(path.join(process.cwd(), './Pulumi.yaml'), 'utf8'),
    )

    if (!doc.lock || !doc.lock.region || !doc.lock.table) {
        console.error(`Set lock config in Pulumi.yaml

lock:
  region: ap-southeast-2
  table: my-table`)
    }
    const dynamodb = new AWS.DynamoDB.DocumentClient({
        region: doc.lock.region,
    })

    const failClosedClient = new DynamoDBLockClient.FailClosed({
        dynamodb,
        lockTable: doc.lock.table,
        partitionKey: 'id',

        acquirePeriodMs: 5000,
        // Retry for a minute
        retryCount: 12,
    })

    const { stdout, exitCode } = await execa('pulumi', ['stack', '--show-name'])
    const stackName = stdout.split(/\r?\n/)[0]
    if (!stackName) {
        console.error('Select stack with pulumi stack select first')
        process.exit(1)
    }

    console.log(`Aquiring lock`)
    failClosedClient.acquireLock(stackName, async (error, lock) => {
        lock.on('error', (lockError) =>
            console.error('failed to heartbeat!', lockError),
        )

        if (error) {
            console.error('error', error)
            process.exit(exitCode)
        }
        console.log(`Aquired lock`)
        let pulumiExitCode = 0

        const pulumiSubProcess = execa('pulumi', pulumiArgs, {
            stdio: [process.stdin, process.stdout, process.stderr],
        })
        try {
            await pulumiSubProcess
        } catch (err) {
            // Just propagate the exit code, pulumi will display the error
            pulumiExitCode = pulumiSubProcess.exitCode
        }

        lock.release((error) => {
            if (error) {
                console.error(error)
            }

            console.log('Released lock')
            process.exit(pulumiExitCode)
        })
    })
})()

Pulumi.yaml

name: serverless-mono
description: Serverless mono infrastructure
backend:
  url: s3://my-pulumi-state-bucket
runtime:
  name: nodejs
  options:
    typescript: false
lock:
  region: ap-southeast-2
  table: my-dynamo-db-table-for-locks

Table.ts

const stateLockTable = new aws.dynamodb.Table(`pulumi-state-lock`, {
    name: `my-dynamo-db-table-for-locks`,
    attributes: [
        {
            name: 'id',
            type: 'S',
        },
    ],
    hashKey: 'id',
    billingMode: 'PAY_PER_REQUEST',
})