s3-list-metadata.go

gistfile1.txt

package main
  
import (
    "fmt"
    "os"
    "sync"
    "time"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/awserr"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
)

func listToChannelAndClose(svc *s3.S3, bucketname *string, pfix string, channel chan string) {
    err := svc.ListObjectsPages(&s3.ListObjectsInput{
        Bucket: bucketname,
        Prefix: &pfix,
    }, func(p *s3.ListObjectsOutput, _ bool) (shouldContinue bool) {
        for _, v := range p.Contents {
            channel <- *v.Key
        }
        return true
    })
    reportAWSError(err)
    close(channel)
}

func main() {
    endpointUrl := os.Getenv("S3_ENDPOINT_URL")
    bucketname := "joshuarobinson"
    prefix := ""
    workerCount := 32

    httpClient, err := NewHTTPClientWithSettings(HTTPClientSettings{
        Connect:          5 * time.Second,
        ExpectContinue:   1 * time.Second,
        IdleConn:         90 * time.Second,
        ConnKeepAlive:    30 * time.Second,
        MaxAllIdleConns:  100,
        MaxHostIdleConns: 100,
        ResponseHeader:   5 * time.Second,
        TLSHandshake:     5 * time.Second,
    })
    if err != nil {
        fmt.Println("Got an error creating custom HTTP client:")
        fmt.Println(err)
        os.Exit(1)
    }

    s3Config := &aws.Config{
        Region:           aws.String("us-east-1"),
        DisableSSL:       aws.Bool(true),
        S3ForcePathStyle: aws.Bool(true),
        HTTPClient:       httpClient,
    }
    if endpointUrl != "" {
        s3Config.Endpoint = &endpointUrl
    }

    sess := session.Must(session.NewSession(s3Config))
    svc := s3.New(sess)

    channel := make(chan string, 1024)
    outstanding := 0
    outstanding++
    go listToChannelAndClose(svc, &bucketname, prefix, channel)

    var wg sync.WaitGroup
    workerFunc := func() {
        defer wg.Done()

        for k := range channel {
            //fmt.Println("Reading object", k)
            input := &s3.HeadObjectInput{
                Bucket: &bucketname,
                Key:    &k,
            }

            res, err := svc.HeadObject(input)
            reportAWSError(err)
            if len(res.Metadata) > 0 {
                fmt.Println(res)
            }
        }
    }

    wg.Add(workerCount)
    for i := 0; i < workerCount; i++ {
        go workerFunc()
    }
    wg.Wait()
}