delete all objects from s3 bucket -- go

deleteAll.go

package main

import (
	"fmt"
	"time"
	"os"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/aws/session"
	"runtime"
)

func getAllObjects(bucketName string, svc *s3.S3) (objects []*s3.ObjectIdentifier){

	fmt.Println("Start getting all objects from bucket", bucketName)

	allObjects := []*s3.ObjectIdentifier{}

	var marker string

	for {
		resp, err := svc.ListObjects(&s3.ListObjectsInput{Bucket: aws.String(bucketName), Marker: &marker, MaxKeys: aws.Int64(1000)})

		if err != nil {
			fmt.Println("Error happend when list all objects.")
			fmt.Println(err.Error())
			return
		}

		for _, v := range resp.Contents {
			oi := &s3.ObjectIdentifier{Key: v.Key}
			allObjects = append(allObjects, oi)
		}

		if *resp.IsTruncated == true {
			marker = *allObjects[len(allObjects) - 1].Key
			fmt.Println("new marker:", marker)
		} else {
			break
		}
	}

	fmt.Println("Finish getting all objects.")

	return allObjects
}


func deleteAllObjects(objects []*s3.ObjectIdentifier, bucket string, svc *s3.S3, c chan int) {
	params := s3.DeleteObjectsInput{
		Bucket: aws.String(bucket),
		Delete: &s3.Delete{Objects: objects},
	}

	fmt.Println("Ready to delete", len(objects))

	out, err := svc.DeleteObjects(&params)

	if err != nil {
		fmt.Println("Error happend when delete objects:\n", err.Error())
		c <- 0
	}

	c <- len(out.Deleted)
	fmt.Println("finish channel write.")
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s bucketName\n", os.Args[0])
	os.Exit(1)
}

func main() {

	args := os.Args[1:]
	if len(args) < 1 {
		usage()
	}
	bucket := args[0]

	runtime.GOMAXPROCS(runtime.NumCPU())

	svc := s3.New(session.New(), &aws.Config{Region: aws.String("ap-northeast-1")})

	objects := getAllObjects(bucket, svc)

	fmt.Println("Total objects:", len(objects))

	routineNumber := len(objects) / 1000

	start := time.Now()
	c := make(chan int, routineNumber + 1)

	fmt.Println("Times:", routineNumber)

	if routineNumber >= 1 {
		for i := 0; i < routineNumber; i++ {
			from := i*1000
			to := (i+1) * 1000
			go deleteAllObjects(objects[from:to], bucket, svc, c)
			fmt.Println("run delete to channel", i)
		}
	}

	fmt.Println("remain objects:", len(objects[routineNumber *1000:]))
	go deleteAllObjects(objects[routineNumber *1000:], bucket, svc, c)

	sum := 0
	i := 0

	L: for {
		select {
		case value := <- c:
			sum += value
			i++
			fmt.Println("i is:", i)
			if i > routineNumber {
				break L
			}
		}
	}

	end := time.Now()
	fmt.Println("Deleted objects count:", sum, "Cost time", end.Sub(start))


}