s3-rapid-list.go

package main
  
import (
    "fmt"
    "os"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

func listprefix(svc *s3.S3, bucketname *string, pfix string, channel chan int) {
    count := 0

    err := svc.ListObjectsPages(&s3.ListObjectsInput{
        Bucket: bucketname,
        Prefix: &pfix,
    }, func(p *s3.ListObjectsOutput, _ bool) (shouldContinue bool) {
        count += len(p.Contents)
        return true
    })
    if err != nil {
        fmt.Println("failed to list objects", err)
        return
    }
    channel <- count
}


func main() {
    if len(os.Args) < 2 {
        fmt.Println("you must specify a bucket")
        return
    }
    bucketname := &os.Args[1]

    s3Config := &aws.Config{
        Endpoint:         aws.String("http://10.62.116.100"),
        Region:           aws.String("us-east-1"),
        DisableSSL:       aws.Bool(true),
        S3ForcePathStyle: aws.Bool(true),
    }

    sess := session.Must(session.NewSession(s3Config))
    svc := s3.New(sess)

    channel := make(chan int)
    outstanding := 0
    prefixlist := "abcdefghijklmnopqrstuvwxyz"
    for _, pfix1 := range prefixlist {
        for _, pfix2 := range prefixlist {
            outstanding++
            go listprefix(svc, bucketname, string(pfix1) + string(pfix2), channel)
        }
    }
    count := 0
    for i := 0; i < outstanding; i++ {
        count += <-channel
    }
    fmt.Println("Count ", count)
}