Created
October 12, 2020 18:41
-
-
Save Dzhuneyt/53d57e1234cafb956791ddcc1ba66406 to your computer and use it in GitHub Desktop.
A CLI script to cleanup S3 buckets that are detached from any CloudFormation stacks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import S3 = require("aws-sdk/clients/s3"); | |
import CloudFormation = require("aws-sdk/clients/cloudformation"); | |
import {StackResourceSummaries, StackSummaries} from "aws-sdk/clients/cloudformation"; | |
import {ObjectVersion} from "aws-sdk/clients/s3"; | |
const s3 = new S3(); | |
const cf = new CloudFormation(); | |
export const emptyBucket = async (bucketName: string) => { | |
// delete bucket contents | |
const listedObjects = await s3 | |
.listObjectVersions({Bucket: bucketName}) | |
.promise() | |
// nothing left - we're done! | |
const contents = (listedObjects.Versions || []).concat( | |
listedObjects.DeleteMarkers || [] | |
) | |
if (contents.length === 0) return | |
let records = [] | |
// make a list of objects to delete | |
for (let record of contents) { | |
records.push({Key: record.Key!, VersionId: record.VersionId!}) | |
} | |
await s3 | |
.deleteObjects({Bucket: bucketName, Delete: {Objects: records}}) | |
.promise() | |
// repeat as necessary | |
if (listedObjects.IsTruncated) await emptyBucket(bucketName) | |
} | |
class CleanupDetachedBuckets { | |
private readonly prefix: string; | |
constructor(prefix: string) { | |
this.prefix = prefix; | |
} | |
/** | |
* | |
*/ | |
async cleanup() { | |
// Delete S3 buckets that are not attached to any CloudFormation resource | |
const bucketNamesToKeep = await this.getBucketsFromMyCloudFormationStacks(); | |
const allBucketsDetachedFromMyCloudFormationStacks = (await this.getS3Buckets()) | |
// Get only buckets that should not be kept (not part of CF stacks) | |
.filter(bucket => bucketNamesToKeep.indexOf(bucket.Name!) === -1); | |
for (const bucketToDelete of allBucketsDetachedFromMyCloudFormationStacks) { | |
await emptyBucket(bucketToDelete.Name!); | |
const deleteBucket = await s3.deleteBucket({ | |
Bucket: bucketToDelete.Name! | |
}).promise(); | |
if (deleteBucket.$response.error) { | |
throw new Error(deleteBucket.$response.error.message); | |
} else { | |
console.log(`Deleted bucket ${bucketToDelete.Name!}`); | |
} | |
} | |
} | |
async emptyBucket(bucketName: string) { | |
console.log(`Emptying bucket ${bucketName}`); | |
const versionedFiles = await s3.listObjectVersions({ | |
Bucket: bucketName, | |
}).promise(); | |
console.log(`Found ${versionedFiles.Versions?.length} versioned files`); | |
if (versionedFiles.Versions?.length) { | |
const deleteVersionedObjectsResult = await s3.deleteObjects({ | |
Bucket: bucketName, | |
Delete: { | |
Objects: versionedFiles.Versions!.map((value: ObjectVersion) => { | |
return { | |
Key: value.Key!, | |
VersionId: value.VersionId!, | |
} | |
}) | |
}, | |
}).promise(); | |
console.log(`Deleted ${deleteVersionedObjectsResult.Deleted!.length} versioned keys from bucket`); | |
if (deleteVersionedObjectsResult.Errors?.length) { | |
console.log('Some errors detected:'); | |
console.log(deleteVersionedObjectsResult.Errors); | |
} | |
} | |
const files = await s3.listObjects({ | |
Bucket: bucketName, | |
}).promise(); | |
console.log(`Found ${files.Contents?.length} files`); | |
if (files.Contents?.length) { | |
const deleteObjectsResult = await s3.deleteObjects({ | |
Bucket: bucketName, | |
Delete: { | |
Objects: files.Contents!.map(value => { | |
return { | |
Key: value.Key!, | |
} | |
}) | |
}, | |
}).promise(); | |
console.log(`Deleted ${deleteObjectsResult.Deleted?.length} keys from bucket`); | |
if (deleteObjectsResult.Errors) { | |
console.log('Some errors detected:'); | |
console.log(deleteObjectsResult.Errors); | |
} | |
} | |
} | |
async getS3Buckets() { | |
const r = await s3.listBuckets().promise(); | |
return (r.Buckets ? r.Buckets : []).filter(bucket => bucket.Name!.startsWith(this.prefix)); | |
} | |
async getStackResources(stackName: string): Promise<StackResourceSummaries> { | |
const iterator = async (NextToken?: string): Promise<StackResourceSummaries> => { | |
const r = await cf.listStackResources({ | |
StackName: stackName, | |
NextToken, | |
}).promise(); | |
if (r.NextToken) { | |
const nextPage = await iterator(r.NextToken); | |
return [ | |
...r.StackResourceSummaries!, | |
...nextPage, | |
]; | |
} | |
return r.StackResourceSummaries!; | |
} | |
return await iterator(); | |
} | |
async getBucketsFromMyCloudFormationStacks(): Promise<string[]> { | |
// Delete S3 buckets that are not attached to any CloudFormation resource | |
const stacks = await this.getCloudFormationStacks(); | |
console.log(`Found ${stacks.length} stacks with prefix ${this.prefix}`); | |
const bucketNames: string[] = []; | |
for (const stack of stacks) { | |
const resources = await this.getStackResources(stack.StackName); | |
resources.forEach(resource => { | |
if (resource.ResourceType === 'AWS::S3::Bucket') { | |
bucketNames.push(resource.PhysicalResourceId!); | |
} | |
}) | |
} | |
console.log(`Found ${bucketNames.length} S3 buckets in my active stacks`); | |
return bucketNames; | |
} | |
async getCloudFormationStacks(): Promise<StackSummaries> { | |
const cf = new CloudFormation(); | |
const iterator = async (token?: string): Promise<StackSummaries> => { | |
const stacks = await cf.listStacks({ | |
NextToken: token, | |
}).promise(); | |
if (stacks.NextToken) { | |
const nextPage = await iterator(stacks.NextToken); | |
return [ | |
...stacks.StackSummaries!, | |
...nextPage, | |
] | |
} | |
return stacks.StackSummaries ? stacks.StackSummaries : []; | |
} | |
// Start paginating recursively | |
const allStacks = await iterator(); | |
return allStacks | |
.filter(stack => stack.StackStatus !== "DELETE_COMPLETE") | |
.filter(stack => stack.StackName.startsWith(this.prefix)); | |
} | |
} | |
const readline = require('readline').createInterface({ | |
input: process.stdin, | |
output: process.stdout | |
}) | |
readline.question(`Please enter a prefix to search buckets to delete:`, (prefix: string) => { | |
new CleanupDetachedBuckets(prefix).cleanup().then(r => { | |
console.log("All completed"); | |
}); | |
readline.close() | |
}) | |
We use this in a project that is managed by AWS CDK, which tends to leave behind a lot of S3 buckets "orphaned". For this reason, we need to run this script periodically, to cleanup old buckets that no CloudFormation stack refers to anymore.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Usage:
npx ts-node cleanup-s3-buckets-detached-from-cloudformation-stacks.ts