Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save Dzhuneyt/53d57e1234cafb956791ddcc1ba66406 to your computer and use it in GitHub Desktop.
Save Dzhuneyt/53d57e1234cafb956791ddcc1ba66406 to your computer and use it in GitHub Desktop.
A CLI script to cleanup S3 buckets that are detached from any CloudFormation stacks
import S3 = require("aws-sdk/clients/s3");
import CloudFormation = require("aws-sdk/clients/cloudformation");
import {StackResourceSummaries, StackSummaries} from "aws-sdk/clients/cloudformation";
import {ObjectVersion} from "aws-sdk/clients/s3";
const s3 = new S3();
const cf = new CloudFormation();
export const emptyBucket = async (bucketName: string) => {
// delete bucket contents
const listedObjects = await s3
.listObjectVersions({Bucket: bucketName})
.promise()
// nothing left - we're done!
const contents = (listedObjects.Versions || []).concat(
listedObjects.DeleteMarkers || []
)
if (contents.length === 0) return
let records = []
// make a list of objects to delete
for (let record of contents) {
records.push({Key: record.Key!, VersionId: record.VersionId!})
}
await s3
.deleteObjects({Bucket: bucketName, Delete: {Objects: records}})
.promise()
// repeat as necessary
if (listedObjects.IsTruncated) await emptyBucket(bucketName)
}
class CleanupDetachedBuckets {
private readonly prefix: string;
constructor(prefix: string) {
this.prefix = prefix;
}
/**
*
*/
async cleanup() {
// Delete S3 buckets that are not attached to any CloudFormation resource
const bucketNamesToKeep = await this.getBucketsFromMyCloudFormationStacks();
const allBucketsDetachedFromMyCloudFormationStacks = (await this.getS3Buckets())
// Get only buckets that should not be kept (not part of CF stacks)
.filter(bucket => bucketNamesToKeep.indexOf(bucket.Name!) === -1);
for (const bucketToDelete of allBucketsDetachedFromMyCloudFormationStacks) {
await emptyBucket(bucketToDelete.Name!);
const deleteBucket = await s3.deleteBucket({
Bucket: bucketToDelete.Name!
}).promise();
if (deleteBucket.$response.error) {
throw new Error(deleteBucket.$response.error.message);
} else {
console.log(`Deleted bucket ${bucketToDelete.Name!}`);
}
}
}
async emptyBucket(bucketName: string) {
console.log(`Emptying bucket ${bucketName}`);
const versionedFiles = await s3.listObjectVersions({
Bucket: bucketName,
}).promise();
console.log(`Found ${versionedFiles.Versions?.length} versioned files`);
if (versionedFiles.Versions?.length) {
const deleteVersionedObjectsResult = await s3.deleteObjects({
Bucket: bucketName,
Delete: {
Objects: versionedFiles.Versions!.map((value: ObjectVersion) => {
return {
Key: value.Key!,
VersionId: value.VersionId!,
}
})
},
}).promise();
console.log(`Deleted ${deleteVersionedObjectsResult.Deleted!.length} versioned keys from bucket`);
if (deleteVersionedObjectsResult.Errors?.length) {
console.log('Some errors detected:');
console.log(deleteVersionedObjectsResult.Errors);
}
}
const files = await s3.listObjects({
Bucket: bucketName,
}).promise();
console.log(`Found ${files.Contents?.length} files`);
if (files.Contents?.length) {
const deleteObjectsResult = await s3.deleteObjects({
Bucket: bucketName,
Delete: {
Objects: files.Contents!.map(value => {
return {
Key: value.Key!,
}
})
},
}).promise();
console.log(`Deleted ${deleteObjectsResult.Deleted?.length} keys from bucket`);
if (deleteObjectsResult.Errors) {
console.log('Some errors detected:');
console.log(deleteObjectsResult.Errors);
}
}
}
async getS3Buckets() {
const r = await s3.listBuckets().promise();
return (r.Buckets ? r.Buckets : []).filter(bucket => bucket.Name!.startsWith(this.prefix));
}
async getStackResources(stackName: string): Promise<StackResourceSummaries> {
const iterator = async (NextToken?: string): Promise<StackResourceSummaries> => {
const r = await cf.listStackResources({
StackName: stackName,
NextToken,
}).promise();
if (r.NextToken) {
const nextPage = await iterator(r.NextToken);
return [
...r.StackResourceSummaries!,
...nextPage,
];
}
return r.StackResourceSummaries!;
}
return await iterator();
}
async getBucketsFromMyCloudFormationStacks(): Promise<string[]> {
// Delete S3 buckets that are not attached to any CloudFormation resource
const stacks = await this.getCloudFormationStacks();
console.log(`Found ${stacks.length} stacks with prefix ${this.prefix}`);
const bucketNames: string[] = [];
for (const stack of stacks) {
const resources = await this.getStackResources(stack.StackName);
resources.forEach(resource => {
if (resource.ResourceType === 'AWS::S3::Bucket') {
bucketNames.push(resource.PhysicalResourceId!);
}
})
}
console.log(`Found ${bucketNames.length} S3 buckets in my active stacks`);
return bucketNames;
}
async getCloudFormationStacks(): Promise<StackSummaries> {
const cf = new CloudFormation();
const iterator = async (token?: string): Promise<StackSummaries> => {
const stacks = await cf.listStacks({
NextToken: token,
}).promise();
if (stacks.NextToken) {
const nextPage = await iterator(stacks.NextToken);
return [
...stacks.StackSummaries!,
...nextPage,
]
}
return stacks.StackSummaries ? stacks.StackSummaries : [];
}
// Start paginating recursively
const allStacks = await iterator();
return allStacks
.filter(stack => stack.StackStatus !== "DELETE_COMPLETE")
.filter(stack => stack.StackName.startsWith(this.prefix));
}
}
const readline = require('readline').createInterface({
input: process.stdin,
output: process.stdout
})
readline.question(`Please enter a prefix to search buckets to delete:`, (prefix: string) => {
new CleanupDetachedBuckets(prefix).cleanup().then(r => {
console.log("All completed");
});
readline.close()
})
@Dzhuneyt
Copy link
Author

Usage: npx ts-node cleanup-s3-buckets-detached-from-cloudformation-stacks.ts

@Dzhuneyt
Copy link
Author

We use this in a project that is managed by AWS CDK, which tends to leave behind a lot of S3 buckets "orphaned". For this reason, we need to run this script periodically, to cleanup old buckets that no CloudFormation stack refers to anymore.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment