comparison of calculating prefix sizes with S3 and Storj native using GC functions

sizes.go

package cloudsigning

import (
	"fmt"
	"net/http"

	"github.com/GoogleCloudPlatform/functions-framework-go/functions"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"

	"storj.io/uplink"
)

const (
	storjS3Id     = "<access key>"
	storjS3Secret = "<secret key>"
	accessGrant   = "<access grant>"
	storjS3URL    = "https://gateway.storjshare.io/"
	bucket        = "files"
)

func init() {
	functions.HTTP("GetSizeS3", GetSizeS3)
	functions.HTTP("GetSizeNative", GetSizeNative)
}

// GetSizeS3 is an HTTP Cloud Function with a request parameter.
func GetSizeS3(w http.ResponseWriter, r *http.Request) {
	prefix := r.URL.Query()["prefix"]
	if len(prefix) == 0 {
		r.Response.StatusCode = 400
		r.Response.Status = "Request is missing 'prefix' query parameter"
		return
	}

	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials.NewStaticCredentials(storjS3Id, storjS3Secret, ""),
		Endpoint:    aws.String(storjS3URL),
		Region:      aws.String("us-east-1"),
	})
	if err != nil {
		r.Response.StatusCode = 500
		r.Response.Status = "Failed to create session"
		return
	}

	svc := s3.New(sess)

	// Call the ListObjectsV2 API to list objects with the specified prefix
	input := &s3.ListObjectsV2Input{
		Bucket: aws.String("files"),
		Prefix: aws.String(prefix[0]),
	}

	totalSize := int64(0)

	err = svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
		for _, obj := range page.Contents {
			totalSize += *obj.Size
		}
		return !lastPage // return true to continue iterating through pages
	})
	if err != nil {
		r.Response.StatusCode = 500
		r.Response.Status = "Error listing S3 objects"
		return
	}

	fmt.Fprintf(w, "Total size of path prefix %s in bucket %s: %d bytes\n", prefix, "files", totalSize)
}

// GetSizeNative is an HTTP Cloud Function with a request parameter.
func GetSizeNative(w http.ResponseWriter, r *http.Request) {
	prefix := r.URL.Query()["prefix"]
	if len(prefix) == 0 {
		r.Response.StatusCode = 400
		r.Response.Status = "Request is missing 'prefix' query parameter"
		return
	}

	access, err := uplink.ParseAccess(accessGrant)
	if err != nil {
		r.Response.StatusCode = 500
		r.Response.Status = "Failed to parse access grant"
		return
	}
	project, err := uplink.OpenProject(r.Context(), access)
	if err != nil {
		r.Response.StatusCode = 500
		r.Response.Status = "Failed to open project"
		return
	}
	defer project.Close()

	listOptions := uplink.ListObjectsOptions{
		Prefix: prefix[0], Recursive: true, System: true, Custom: false,
	}

	totalSize := int64(0)
	objectIterator := project.ListObjects(r.Context(), bucket, &listOptions)
	for objectIterator.Next() {
		item := objectIterator.Item()
		totalSize += item.System.ContentLength
	}

	if err := objectIterator.Err(); err != nil {
		r.Response.StatusCode = 500
		r.Response.Status = "Error while iterating objects"
		return
	}

	fmt.Fprintf(w, "Total size of path prefix %s in bucket %s: %d bytes\n", prefix, "files", totalSize)
}