AWS - S3 - Bypass the 1000 files limit - Golang

aws-list-files-inside-directory.go

package main

import (
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

func awsLink(bucket, directory string) (*s3.S3, *s3.ListObjectsV2Input) {
	// Set up parameters
	cred := credentials.NewStaticCredentials(<secret>, <secret>, "")
	config := aws.NewConfig()
	region := "eu-central-1"
	config.WithCredentials(cred).WithRegion(region)
	// Create a session to share configuration, and load external configuration.
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		Config: *config,
	}))
	// Create S3 service client
	svc := s3.New(sess)
	fmt.Printf("Bucket: %s\n", bucket)
	// Set up parameters
	params := &s3.ListObjectsV2Input{
		Bucket: aws.String(bucket),
		Prefix: aws.String(directory),
	}
	return svc, params
}

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

// Default ListObjectsV2 is limited up to 1000 files
// Using NextContinuationToken and IsTruncated allows
// us to get all the files inside the directory that
// is to say bypass the 1000 limit
func getListFiles(bucket string, svc *s3.S3, params *s3.ListObjectsV2Input) []string {
	var listUrl []string
	truncatedListing := true
	for truncatedListing {
		resp, err := svc.ListObjectsV2(params)
		if err != nil {
			exitErrorf("Unable to list items in bucket %q, %v", bucket, err)
		}
		for _, key := range resp.Contents {
			fmt.Printf("  🔍 Loading... \r")
			listUrl = append(listUrl, *key.Key)
		}
		params.ContinuationToken = resp.NextContinuationToken
		truncatedListing = *resp.IsTruncated
	}
	return listUrl
}

func main() {
	if len(os.Args) != 3 {
		fmt.Printf("Usage: go run aws-list-files-inside-directory.go bucket directory\nBucket: <name>\nDirectory: <name>\n")
		return
	}
	bucket := os.Args[1]
	directory := os.Args[2]
	svc, params := awsLink(bucket, directory)
	listUrl := getListFiles(bucket, svc, params)
	for _, file := range listUrl {
		fmt.Printf("%s\n", file)
	}
	fmt.Println("Sum:", len(listUrl))
}