s3_multipart_upload.go

package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

const (
	CRED_PROFILE = "test-credencial"
	BUCKET_NAME  = "my-bucket-name"
	REGION       = "us-west-2"
)

var partSize int64 = 1024 * 1024 * 8 // 8MB

func uploadMultipart(svc *s3.S3, filename string) error {
	mgr := s3manager.NewUploader(&s3manager.UploadOptions{
		S3:          svc,
		PartSize:    partSize,
		Concurrency: 1,
	})

	f, err := os.Open(filename)
	if err != nil {
		fmt.Println("os.Open:", err)
		return err
	}
	defer f.Close()

	resp, err := mgr.Upload(&s3manager.UploadInput{
		Bucket: aws.String(BUCKET_NAME),
		Key:    aws.String(filename),
		Body:   f,
	})
	if err != nil {
		fmt.Println("s3manager.Upload:", err)
		return err
	}

	fmt.Println(resp)

	return nil
}

func uploadSingle(svc *s3.S3, filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		fmt.Println("os.Open:", err)
		return err
	}
	defer f.Close()

	resp, err := svc.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(BUCKET_NAME),
		Key:    aws.String("dmyPut"),
		Body:   f,
	})
	if err != nil {
		fmt.Println("svc.PutObject:", err)
		return err
	}

	fmt.Println(resp)

	return nil
}

func uploadFile(svc *s3.S3, filename string) error {
	fstat, err := os.Stat(filename)
	if err != nil {
		fmt.Printf("file '%s' not exits!!\n", filename)
		return err
	}

	if fstat.Size() >= partSize {
		fmt.Println("multipart upload")
		uploadMultipart(svc, filename)
	} else {
		fmt.Println("sigle file upload")
		uploadSingle(svc, filename)
	}

	return nil
}

func main() {
	flag.Parse()

	targetFilename := flag.Args()[0]

	creds := credentials.NewSharedCredentials("", CRED_PROFILE)
	svc := s3.New(&aws.Config{
		Region:      aws.String(REGION),
		Credentials: creds,
	})

	uploadFile(svc, targetFilename)

	resp, err := svc.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(BUCKET_NAME),
	})
	if err != nil {
		fmt.Println("ListObjects:", err)
		return
	}
	for _, c := range resp.Contents {
		fmt.Printf("last-modified:%v, key=%v, size=%d\n", c.LastModified, *c.Key, *c.Size)
	}
}

$ mkdir ~/.aws/
$ vim ~/.aws/credentials
[test-credencial]
aws_access_key_id = XXXX
aws_secret_access_key = XXXX
$ go build -o t s3_multipart_upload.go
$ ./t FILE