A CLI script to cleanup S3 buckets that are detached from any CloudFormation stacks

cleanup-s3-buckets-detached-from-cloudformation-stacks.ts

import S3 = require("aws-sdk/clients/s3");
import CloudFormation = require("aws-sdk/clients/cloudformation");
import {StackResourceSummaries, StackSummaries} from "aws-sdk/clients/cloudformation";
import {ObjectVersion} from "aws-sdk/clients/s3";

const s3 = new S3();
const cf = new CloudFormation();

export const emptyBucket = async (bucketName: string) => {
    // delete bucket contents
    const listedObjects = await s3
        .listObjectVersions({Bucket: bucketName})
        .promise()

    // nothing left - we're done!
    const contents = (listedObjects.Versions || []).concat(
        listedObjects.DeleteMarkers || []
    )
    if (contents.length === 0) return

    let records = []

    // make a list of objects to delete
    for (let record of contents) {
        records.push({Key: record.Key!, VersionId: record.VersionId!})
    }

    await s3
        .deleteObjects({Bucket: bucketName, Delete: {Objects: records}})
        .promise()

    // repeat as necessary
    if (listedObjects.IsTruncated) await emptyBucket(bucketName)
}

class CleanupDetachedBuckets {

    private readonly prefix: string;

    constructor(prefix: string) {
        this.prefix = prefix;
    }

    /**
     *
     */
    async cleanup() {
        // Delete S3 buckets that are not attached to any CloudFormation resource
        const bucketNamesToKeep = await this.getBucketsFromMyCloudFormationStacks();

        const allBucketsDetachedFromMyCloudFormationStacks = (await this.getS3Buckets())
            // Get only buckets that should not be kept (not part of CF stacks)
            .filter(bucket => bucketNamesToKeep.indexOf(bucket.Name!) === -1);
        for (const bucketToDelete of allBucketsDetachedFromMyCloudFormationStacks) {
            await emptyBucket(bucketToDelete.Name!);
            const deleteBucket = await s3.deleteBucket({
                Bucket: bucketToDelete.Name!
            }).promise();
            if (deleteBucket.$response.error) {
                throw new Error(deleteBucket.$response.error.message);
            } else {
                console.log(`Deleted bucket ${bucketToDelete.Name!}`);
            }
        }
    }

    async emptyBucket(bucketName: string) {
        console.log(`Emptying bucket ${bucketName}`);

        const versionedFiles = await s3.listObjectVersions({
            Bucket: bucketName,
        }).promise();
        console.log(`Found ${versionedFiles.Versions?.length} versioned files`);
        if (versionedFiles.Versions?.length) {
            const deleteVersionedObjectsResult = await s3.deleteObjects({
                Bucket: bucketName,
                Delete: {
                    Objects: versionedFiles.Versions!.map((value: ObjectVersion) => {
                        return {
                            Key: value.Key!,
                            VersionId: value.VersionId!,
                        }
                    })
                },
            }).promise();

            console.log(`Deleted ${deleteVersionedObjectsResult.Deleted!.length} versioned keys from bucket`);
            if (deleteVersionedObjectsResult.Errors?.length) {
                console.log('Some errors detected:');
                console.log(deleteVersionedObjectsResult.Errors);
            }
        }

        const files = await s3.listObjects({
            Bucket: bucketName,
        }).promise();
        console.log(`Found ${files.Contents?.length} files`);
        if (files.Contents?.length) {
            const deleteObjectsResult = await s3.deleteObjects({
                Bucket: bucketName,
                Delete: {
                    Objects: files.Contents!.map(value => {
                        return {
                            Key: value.Key!,
                        }
                    })
                },
            }).promise();

            console.log(`Deleted ${deleteObjectsResult.Deleted?.length} keys from bucket`);
            if (deleteObjectsResult.Errors) {
                console.log('Some errors detected:');
                console.log(deleteObjectsResult.Errors);
            }
        }


    }

    async getS3Buckets() {
        const r = await s3.listBuckets().promise();
        return (r.Buckets ? r.Buckets : []).filter(bucket => bucket.Name!.startsWith(this.prefix));
    }

    async getStackResources(stackName: string): Promise<StackResourceSummaries> {
        const iterator = async (NextToken?: string): Promise<StackResourceSummaries> => {
            const r = await cf.listStackResources({
                StackName: stackName,
                NextToken,
            }).promise();
            if (r.NextToken) {
                const nextPage = await iterator(r.NextToken);
                return [
                    ...r.StackResourceSummaries!,
                    ...nextPage,
                ];
            }
            return r.StackResourceSummaries!;
        }

        return await iterator();
    }

    async getBucketsFromMyCloudFormationStacks(): Promise<string[]> {
        // Delete S3 buckets that are not attached to any CloudFormation resource
        const stacks = await this.getCloudFormationStacks();
        console.log(`Found ${stacks.length} stacks with prefix ${this.prefix}`);
        const bucketNames: string[] = [];
        for (const stack of stacks) {
            const resources = await this.getStackResources(stack.StackName);
            resources.forEach(resource => {
                if (resource.ResourceType === 'AWS::S3::Bucket') {
                    bucketNames.push(resource.PhysicalResourceId!);
                }
            })
        }

        console.log(`Found ${bucketNames.length} S3 buckets in my active stacks`);

        return bucketNames;
    }

    async getCloudFormationStacks(): Promise<StackSummaries> {
        const cf = new CloudFormation();

        const iterator = async (token?: string): Promise<StackSummaries> => {
            const stacks = await cf.listStacks({
                NextToken: token,
            }).promise();
            if (stacks.NextToken) {
                const nextPage = await iterator(stacks.NextToken);
                return [
                    ...stacks.StackSummaries!,
                    ...nextPage,
                ]
            }
            return stacks.StackSummaries ? stacks.StackSummaries : [];
        }

        // Start paginating recursively
        const allStacks = await iterator();
        return allStacks
            .filter(stack => stack.StackStatus !== "DELETE_COMPLETE")
            .filter(stack => stack.StackName.startsWith(this.prefix));
    }
}

const readline = require('readline').createInterface({
    input: process.stdin,
    output: process.stdout
})

readline.question(`Please enter a prefix to search buckets to delete:`, (prefix: string) => {
    new CleanupDetachedBuckets(prefix).cleanup().then(r => {
        console.log("All completed");
    });

    readline.close()
})

Usage: npx ts-node cleanup-s3-buckets-detached-from-cloudformation-stacks.ts

We use this in a project that is managed by AWS CDK, which tends to leave behind a lot of S3 buckets "orphaned". For this reason, we need to run this script periodically, to cleanup old buckets that no CloudFormation stack refers to anymore.