Skip to content

Instantly share code, notes, and snippets.

@liul85
Created December 25, 2015 04:14
Show Gist options
  • Save liul85/436876ccae0aa96b2860 to your computer and use it in GitHub Desktop.
Save liul85/436876ccae0aa96b2860 to your computer and use it in GitHub Desktop.
delete all objects from s3 bucket -- go
package main
import (
"fmt"
"time"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/aws/session"
"runtime"
)
func getAllObjects(bucketName string, svc *s3.S3) (objects []*s3.ObjectIdentifier){
fmt.Println("Start getting all objects from bucket", bucketName)
allObjects := []*s3.ObjectIdentifier{}
var marker string
for {
resp, err := svc.ListObjects(&s3.ListObjectsInput{Bucket: aws.String(bucketName), Marker: &marker, MaxKeys: aws.Int64(1000)})
if err != nil {
fmt.Println("Error happend when list all objects.")
fmt.Println(err.Error())
return
}
for _, v := range resp.Contents {
oi := &s3.ObjectIdentifier{Key: v.Key}
allObjects = append(allObjects, oi)
}
if *resp.IsTruncated == true {
marker = *allObjects[len(allObjects) - 1].Key
fmt.Println("new marker:", marker)
} else {
break
}
}
fmt.Println("Finish getting all objects.")
return allObjects
}
func deleteAllObjects(objects []*s3.ObjectIdentifier, bucket string, svc *s3.S3, c chan int) {
params := s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &s3.Delete{Objects: objects},
}
fmt.Println("Ready to delete", len(objects))
out, err := svc.DeleteObjects(&params)
if err != nil {
fmt.Println("Error happend when delete objects:\n", err.Error())
c <- 0
}
c <- len(out.Deleted)
fmt.Println("finish channel write.")
}
func usage() {
fmt.Fprintf(os.Stderr, "Usage: %s bucketName\n", os.Args[0])
os.Exit(1)
}
func main() {
args := os.Args[1:]
if len(args) < 1 {
usage()
}
bucket := args[0]
runtime.GOMAXPROCS(runtime.NumCPU())
svc := s3.New(session.New(), &aws.Config{Region: aws.String("ap-northeast-1")})
objects := getAllObjects(bucket, svc)
fmt.Println("Total objects:", len(objects))
routineNumber := len(objects) / 1000
start := time.Now()
c := make(chan int, routineNumber + 1)
fmt.Println("Times:", routineNumber)
if routineNumber >= 1 {
for i := 0; i < routineNumber; i++ {
from := i*1000
to := (i+1) * 1000
go deleteAllObjects(objects[from:to], bucket, svc, c)
fmt.Println("run delete to channel", i)
}
}
fmt.Println("remain objects:", len(objects[routineNumber *1000:]))
go deleteAllObjects(objects[routineNumber *1000:], bucket, svc, c)
sum := 0
i := 0
L: for {
select {
case value := <- c:
sum += value
i++
fmt.Println("i is:", i)
if i > routineNumber {
break L
}
}
}
end := time.Now()
fmt.Println("Deleted objects count:", sum, "Cost time", end.Sub(start))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment