Skip to content

Instantly share code, notes, and snippets.

@brunoksato
Created June 27, 2023 14:55
Show Gist options
  • Save brunoksato/e6c5dc8a0adee8dfe5fd7fee990a6cff to your computer and use it in GitHub Desktop.
Save brunoksato/e6c5dc8a0adee8dfe5fd7fee990a6cff to your computer and use it in GitHub Desktop.
package main
import (
"fmt"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
func exitErrorf(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
}
func main() {
bucket := "prod.assets.onquidd.com"
sess, _ := session.NewSession(&aws.Config{
Region: aws.String("us-east-1")},
)
svc := s3.New(sess)
totalIterated := 0
input := &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String("shelfies/uploads"),
}
total := 0
pageNum := 0
objects := []s3manager.BatchDeleteObject{}
err := svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
pageNum++
for _, item := range page.Contents {
if strings.Contains(*item.Key, "-share.jpg") || strings.Contains(*item.Key, "-share.png") || strings.Contains(*item.Key, "@w") {
objects = append(objects, s3manager.BatchDeleteObject{
Object: &s3.DeleteObjectInput{
Key: item.Key,
Bucket: aws.String(bucket),
},
})
}
}
if len(objects) >= 1000 {
start := time.Now()
batcher := s3manager.NewBatchDelete(sess, func(bd *s3manager.BatchDelete) {
bd.BatchSize = 1000
})
if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
Objects: objects,
}); err != nil {
exitErrorf("Unable to delete items in bucket %q, %v", bucket, err)
}
total += len(objects)
elapsed := time.Since(start)
fmt.Println("Total deleted ", total, "items from bucket", bucket, " took: ", elapsed)
objects = []s3manager.BatchDeleteObject{}
}
totalIterated += len(page.Contents)
fmt.Println("Total iterated", totalIterated, "items from bucket", bucket)
return !lastPage
})
if err != nil {
exitErrorf("Unable to list items in bucket %q, %v", bucket, err)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment