cloud.google.com/go/storage v1.31.0

README

1. Create a "test_data" subdirectory and put a "gopher.png" in it.  (We use the Go mascot but contents don't matter.)
2. Edit the `cloudstorage_test.go` value for `testBucket` to a real Google Cloud Storage bucket name that will be modified.

```
$ for i in v1.30.1 v1.31.0; do go get cloud.google.com/go/storage@$i; GOOGLE_APPLICATION_CREDENTIALS=replay-service-account.json go test --run TestCloudStorage --cloudStorageRecord ./... && go test ./...; done
go: downgraded cloud.google.com/go/storage v1.31.0 => v1.30.1
PASS
ok  	example.com/cloudstorage	2.931s
ok  	example.com/cloudstorage	7.104s
go: upgraded cloud.google.com/go/storage v1.30.1 => v1.31.0
PASS
ok  	example.com/cloudstorage	2.892s
2023/07/07 12:10:21 ERROR: martian: failed to round trip: no matching request for [...]
```

cloudstorage.go

package cloudstorage

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sort"
	"time"

	"cloud.google.com/go/httpreplay"
	"cloud.google.com/go/storage"
	"github.com/go-kit/log"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
)

type CloudStorage struct {
	bucket *storage.BucketHandle
	logger log.Logger
}

func NewCloudStorage(client *storage.Client, bucketName string, logger log.Logger) *CloudStorage {
	bkt := client.Bucket(bucketName)
	return &CloudStorage{
		bucket: bkt,
		logger: log.With(logger, "source", "GoogleCloudStorage"),
	}
}

func (cs *CloudStorage) Write(ctx context.Context, fileName string, object []byte) error {
	var err error
	wc := cs.bucket.Object(fileName).NewWriter(ctx)
	if size, errWrite := wc.Write(object); errWrite != nil {
		err = fmt.Errorf("createFile: unable to write data to bucket %v, file %v: %w", cs.bucket, fileName, errWrite)
	} else if size != len(object) {
		err = fmt.Errorf("createFile: unable to write data to bucket %v, file %v: size %d written different than source size %d", cs.bucket, fileName, size, len(object))
	}
	if errClose := wc.Close(); err == nil && errClose != nil {
		err = fmt.Errorf("createFile: error on closing write to bucket %v, file %v: %w", cs.bucket, fileName, errClose)
	}
	return err
}

func (cs *CloudStorage) UpdatedTime(ctx context.Context, fileName string) (time.Time, error) {
	g, err := cs.bucket.Object(fileName).Attrs(ctx)
	if err != nil {
		return time.Time{}, err
	}
	return g.Updated, nil
}

func (cs *CloudStorage) Read(ctx context.Context, fileName string) ([]byte, error) {
	rc, err := cs.bucket.Object(fileName).NewReader(ctx)
	if err != nil {
		return nil, err
	}
	defer rc.Close()
	return io.ReadAll(rc)
}

func (cs *CloudStorage) Delete(ctx context.Context, fileName string) error {
	return cs.bucket.Object(fileName).Delete(ctx)
}

func (cs *CloudStorage) List(ctx context.Context, prefix string) *storage.ObjectIterator {
	return cs.bucket.Objects(ctx, &storage.Query{Delimiter: "/", Prefix: prefix})
}

func (cs *CloudStorage) Copy(ctx context.Context, srcFile, dstFile string, dstBucket *storage.BucketHandle) error {
	src := cs.bucket.Object(srcFile)
	dst := dstBucket.Object(dstFile)
	copier := dst.CopierFrom(src)
	_, err := copier.Run(ctx)
	return err
}

func (cs *CloudStorage) SortedList(ctx context.Context, prefix string) ([]*storage.ObjectAttrs, error) {
	objs, err := IterateAll(cs.List(ctx, prefix))
	if err != nil {
		return nil, err
	}
	byDate := UpdateDateOrderedObjects(objs)
	sort.Sort(byDate)
	return byDate, nil
}

type UpdateDateOrderedObjects []*storage.ObjectAttrs

func (u UpdateDateOrderedObjects) Len() int           { return len(u) }
func (u UpdateDateOrderedObjects) Less(i, j int) bool { return u[i].Updated.Before(u[j].Updated) }
func (u UpdateDateOrderedObjects) Swap(i, j int)      { u[i], u[j] = u[j], u[i] }

func IterateAll(it *storage.ObjectIterator) ([]*storage.ObjectAttrs, error) {
	objs := make([]*storage.ObjectAttrs, 0)
	for {
		obj, err := it.Next()
		if errors.Is(err, iterator.Done) {
			break
		}
		if err != nil {
			return nil, err
		}
		objs = append(objs, obj)
	}
	return objs, nil
}

type FatalFer interface {
	Fatalf(string, ...interface{})
}

func TestCloudStorageClient(t FatalFer, recordingPath string, record bool) (*storage.Client, io.Closer) {
	var c io.Closer
	var err error
	var testClient *storage.Client
	if record {
		testClient, c, err = newRecording(recordingPath)
	} else {
		testClient, c, err = replayRecording(recordingPath)
	}
	if err != nil {
		t.Fatalf("failed to start replay/record: %v", err)
	}
	return testClient, c
}

func newRecording(replayFile string) (*storage.Client, io.Closer, error) {
	rec, err := httpreplay.NewRecorder(replayFile, []byte(fmt.Sprintf(`{"Date": "%s"`, time.Now().UTC())))
	if err != nil {
		return nil, nil, fmt.Errorf("failed to initialize recorder: %w", err)
	}
	ctx := context.Background()
	creds, err := google.FindDefaultCredentials(ctx, storage.ScopeReadWrite)
	if err != nil {
		return nil, nil, err
	}
	rc, err := rec.Client(ctx, option.WithCredentials(creds))
	if err != nil {
		return nil, nil, err
	}
	testClient, err := storage.NewClient(ctx, option.WithHTTPClient(rc))
	if err != nil {
		return nil, nil, fmt.Errorf("error connecting new client: %w", err)
	}
	return testClient, rec, nil
}

func replayRecording(replayFile string) (*storage.Client, io.Closer, error) {
	rep, err := httpreplay.NewReplayer(replayFile)
	if err != nil {
		return nil, nil, err
	}
	ctx := context.Background()
	rc, err := rep.Client(ctx)
	if err != nil {
		return nil, nil, err
	}
	testClient, err := storage.NewClient(ctx, option.WithHTTPClient(rc))
	if err != nil {
		return nil, nil, fmt.Errorf("error connecting new client: %w", err)
	}
	return testClient, rep, nil
}

cloudstorage_test.go

package cloudstorage

import (
	"context"
	"flag"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/go-kit/log"
)

const (
	testBucket  = "SomeRealBucketHere"
	testDataDir = "test_data/"
)

var (
	record        = flag.Bool("cloudStorageRecord", false, "Create a new recording")
	testFiles     = []string{"tmp-test-file.png", "tmp-test-file2.png"}
	testCopyFiles = []string{"tmp-test-file-copy.png", "tmp-test-file2-copy.png"}
)

func TestCloudStorage(t *testing.T) {
	ctx := context.Background()
	logger := log.NewNopLogger()

	recordingPath, err := filepath.Abs(testDataDir + "cloudstorage.replay")
	if err != nil {
		t.Fatalf("TestCloudStorage - failed to read file: %v", err)
	}
	recordingPathCopy, err := filepath.Abs(testDataDir + "cloudstorageCopy.replay")
	if err != nil {
		t.Fatalf("TestCloudStorage - failed to read file: %v", err)
	}

	testClient, c := TestCloudStorageClient(t, recordingPath, *record)
	defer c.Close()
	cs := NewCloudStorage(testClient, testBucket, logger)

	testCopyClient, c2 := TestCloudStorageClient(t, recordingPathCopy, *record)
	defer c2.Close()
	csCopy := NewCloudStorage(testCopyClient, testBucket, logger)

	obj, err := os.ReadFile(testDataDir + "gopher.png")
	if err != nil {
		t.Fatalf("TestCloudStorage - failed to read file: %v", err)
	}

	var wantListing []string
	for _, name := range testFiles {
		path := filepath.Join(testDataDir, name)
		wantListing = append(wantListing, path)
		if err := cs.Write(ctx, path, obj); err != nil {
			t.Fatalf("TestCloudStorage - failed to write object: %v", err)
		}

		s := strings.Split(path, ".")
		copyPath := s[0] + "-copy." + s[1]
		err = cs.Copy(ctx, path, copyPath, csCopy.bucket)
		if err != nil {
			t.Fatal(err)
		}
		wantListing = append(wantListing, copyPath)

		updatedTime, err := cs.UpdatedTime(ctx, path)
		if err != nil {
			t.Fatal(err)
		}
		if updatedTime.IsZero() {
			t.Error("updated time for the object was expected to be not zero")
		}
		r, err := cs.Read(ctx, path)
		if err != nil {
			t.Fatalf("TestCloudStorage - failed to read object: %v", err)
		}

		if string(obj) != string(r) {
			t.Errorf("TestCloudStorage - object in storage does not match expected file")
		}
	}

	gotFull, err := cs.SortedList(ctx, "test_data/")
	if err != nil {
		t.Errorf("Failed listing the top level dir: %v", err)
	}
	got := make([]string, len(gotFull))
	for i, obj := range gotFull {
		got[i] = obj.Name
	}
	if !reflect.DeepEqual(got, wantListing) {
		t.Errorf("Top level dir listing got %+v, want %+v", got, wantListing)
	}

	for _, name := range testFiles {
		path := filepath.Join(testDataDir, name)
		if err := cs.Delete(ctx, path); err != nil {
			t.Errorf("TestCloudStorage - failed to delete object: %v", err)
		}
	}
	for _, name := range testCopyFiles {
		path := filepath.Join(testDataDir, name)
		if err := cs.Delete(ctx, path); err != nil {
			t.Errorf("TestCloudStorage - failed to delete object: %v", err)
		}
	}
}

go.mod

module example.com/cloudstorage

go 1.20

require (
	cloud.google.com/go v0.110.4
	cloud.google.com/go/storage v1.31.0
	github.com/go-kit/log v0.2.1
	github.com/hashicorp/go-cleanhttp v0.5.2
	golang.org/x/oauth2 v0.10.0
	google.golang.org/api v0.130.0
)

require (
	cloud.google.com/go/compute v1.20.1 // indirect
	cloud.google.com/go/compute/metadata v0.2.3 // indirect
	cloud.google.com/go/iam v1.1.0 // indirect
	github.com/go-logfmt/logfmt v0.5.1 // indirect
	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
	github.com/golang/protobuf v1.5.3 // indirect
	github.com/google/go-cmp v0.5.9 // indirect
	github.com/google/martian/v3 v3.3.2 // indirect
	github.com/google/s2a-go v0.1.4 // indirect
	github.com/google/uuid v1.3.0 // indirect
	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
	go.opencensus.io v0.24.0 // indirect
	golang.org/x/crypto v0.11.0 // indirect
	golang.org/x/net v0.12.0 // indirect
	golang.org/x/sys v0.10.0 // indirect
	golang.org/x/text v0.11.0 // indirect
	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
	google.golang.org/appengine v1.6.7 // indirect
	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
	google.golang.org/grpc v1.56.1 // indirect
	google.golang.org/protobuf v1.31.0 // indirect
)