s3seeker.go

package main

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/jszwec/s3fs"
)

var (
	endpoint   = flag.String("endpoint", "localhost:4566", "s3 endpoint")
	bucket     = flag.String("bucket", "test-github.com-jszwec-s3fs", "bucket name")
	skipVerify = flag.Bool("skip-verify", true, "http insecure skip verify")
)

var (
	accessKeyID = envDefault("S3FS_TEST_AWS_ACCESS_KEY_ID", "1")
	secretKey   = envDefault("S3FS_TEST_AWS_SECRET_ACCESS_KEY", "1")
	region      = envDefault("S3FS_TEST_AWS_REGION", "us-east-1")
)

func main() {
	flag.Parse()

	var osfsys osFS
	s3fsys, err := newS3FS(*bucket)
	if err != nil {
		panic("failed to set up s3 client: " + err.Error())
	}

	var (
		old = []byte("original content")
		new = []byte("w00t")
	)

	for _, tt := range []struct {
		desc string
		fsys FS
	}{
		{"os file system", osfsys},
		{"s3 fsys", s3fsys},
	} {
		if err := doTest(tt.fsys, old, new); err != nil {
			log.Printf("%s failed: %v", tt.desc, err)
			continue
		}
		log.Printf("%s ok", tt.desc)
	}

}

func doTest(fsys FS, old, new []byte) error {
	const filename = "s3fs-test.txt"

	if err := fsys.Write(filename, old); err != nil {
		return err
	}
	defer fsys.Remove(filename)

	f, err := fsys.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	fseeker, ok := f.(io.ReadSeekCloser)
	if ok {
		if _, err := fseeker.Seek(0, io.SeekEnd); err != nil {
			return err
		}
	}

	if err := fsys.Remove(filename); err != nil {
		return err
	}

	if err := fsys.Write(filename, new); err != nil {
		return err
	}

	if ok {
		if _, err := fseeker.Seek(0, io.SeekStart); err != nil {
			return err
		}
	}

	b, err := io.ReadAll(f)
	if err != nil {
		return err
	}

	if !bytes.Equal(old, b) {
		return fmt.Errorf("expected %q; got %q", old, b)
	}

	return nil
}

type FS interface {
	Open(string) (io.ReadCloser, error)
	Write(string, []byte) error
	Remove(string) error
}

type osFS struct{}

func (fs osFS) Open(path string) (io.ReadCloser, error) {
	return os.Open(path)
}

func (fs osFS) Write(path string, data []byte) error {
	return ioutil.WriteFile(path, data, 0655)
}

func (fs osFS) Remove(path string) error {
	return os.RemoveAll(path)
}

type s3FS struct {
	cl     s3iface.S3API
	bucket string
	s3fs   *s3fs.S3FS
}

func newS3FS(bucket string) (*s3FS, error) {
	cl, err := newClient()
	if err != nil {
		return nil, err
	}
	if err := createBucket(cl, bucket); err != nil {
		return nil, err
	}

	return &s3FS{
		cl:     cl,
		bucket: bucket,
		s3fs:   s3fs.New(cl, bucket),
	}, nil
}

func (fs s3FS) Open(path string) (io.ReadCloser, error) {
	return fs.s3fs.Open(path)
}

func (fs s3FS) Write(name string, data []byte) error {
	uploader := s3manager.NewUploaderWithClient(fs.cl)
	_, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bytes.NewReader(data),
		Bucket: &fs.bucket,
		Key:    &name,
	})
	return err
}

func (fs s3FS) Remove(path string) error {
	_, err := fs.cl.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &fs.bucket,
		Key:    &path,
	})
	return err
}

func createBucket(cl s3iface.S3API, bucket string) error {
	_, err := cl.CreateBucket(&s3.CreateBucketInput{
		Bucket: &bucket,
	})
	return err
}

func newClient() (s3iface.S3API, error) {
	cl := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: *skipVerify,
			},
		},
	}

	s, err := session.NewSession(
		aws.NewConfig().
			WithEndpoint(*endpoint).
			WithRegion(region).
			WithS3ForcePathStyle(true).
			WithHTTPClient(cl).
			WithCredentials(credentials.NewStaticCredentials(accessKeyID, secretKey, "")),
	)
	if err != nil {
		return nil, err
	}
	return s3.New(s), nil
}

func envDefault(env, def string) string {
	if os.Getenv(env) == "" {
		return def
	}
	return os.Getenv(env)
}