Created
September 3, 2018 10:57
-
-
Save artyom/9cc5cafda1dccae92d4cb5f83a55b120 to your computer and use it in GitHub Desktop.
upload-s3-random-data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| module github.com/artyom/upload-s3-random-data | |
| require ( | |
| github.com/aws/aws-sdk-go v1.15.6 | |
| github.com/davecgh/go-spew v1.1.0 // indirect | |
| github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f // indirect | |
| github.com/jtolds/gls v4.2.1+incompatible // indirect | |
| github.com/pmezard/go-difflib v1.0.0 // indirect | |
| github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 // indirect | |
| github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect | |
| github.com/stretchr/testify v1.2.2 // indirect | |
| golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24 // indirect | |
| golang.org/x/text v0.3.0 // indirect | |
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| github.com/aws/aws-sdk-go v1.15.6 h1:JRgUEp143FVHH3LOV2ggH2f6h+Tl1EgGphqaoH/Xibk= | |
| github.com/aws/aws-sdk-go v1.15.6/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= | |
| github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= | |
| github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |
| github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= | |
| github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= | |
| github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f h1:FDM3EtwZLyhW48YRiyqjivNlNZjAObv4xt4NnJaU+NQ= | |
| github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= | |
| github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= | |
| github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= | |
| github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= | |
| github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= | |
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | |
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |
| github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 h1:lXQ+j+KwZcbwrbgU0Rp4Eglg3EJLHbuZU3BbOqAGBmg= | |
| github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= | |
| github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo= | |
| github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= | |
| github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= | |
| github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= | |
| golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24 h1:mEsFm194MmS9vCwxFy+zwu0EU7ZkxxMD1iH++vmGdUY= | |
| golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | |
| golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= | |
| golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Program creates 4 test files with pseudorandom data in given s3 bucket | |
| package main | |
| import ( | |
| "context" | |
| "errors" | |
| "flag" | |
| "fmt" | |
| "io" | |
| "math/rand" | |
| "os" | |
| "time" | |
| "github.com/aws/aws-sdk-go/aws" | |
| "github.com/aws/aws-sdk-go/aws/session" | |
| "github.com/aws/aws-sdk-go/service/s3" | |
| "github.com/aws/aws-sdk-go/service/s3/s3manager" | |
| ) | |
| func main() { | |
| var bucket string | |
| flag.StringVar(&bucket, "bucket", bucket, "s3 bucket to create files") | |
| flag.Parse() | |
| if err := run(bucket); err != nil { | |
| os.Stderr.WriteString(err.Error() + "\n") | |
| os.Exit(1) | |
| } | |
| } | |
| func run(bucket string) error { | |
| if bucket == "" { | |
| return errors.New("empty bucket name") | |
| } | |
| sess, err := session.NewSession() | |
| if err != nil { | |
| return err | |
| } | |
| ctx := context.Background() | |
| uploader := s3manager.NewUploader(sess) | |
| files := []struct { | |
| key string | |
| size int64 | |
| }{ | |
| {"testfile/small", 8 << 20}, | |
| {"testfile/medium", 64 << 20}, | |
| {"testfile/large", 256 << 20}, | |
| {"testfile/giant", 1024 << 20}, | |
| } | |
| for _, f := range files { | |
| start := time.Now() | |
| err := uploadRandomFile(ctx, uploader, bucket, f.key, f.size) | |
| if err != nil { | |
| return err | |
| } | |
| fmt.Printf("uploaded %q (%dmb) in %v\n", f.key, f.size>>20, | |
| time.Since(start).Round(10*time.Millisecond)) | |
| } | |
| return nil | |
| } | |
| func uploadRandomFile(ctx context.Context, upl *s3manager.Uploader, bucket, key string, size int64) error { | |
| rnd := rand.New(rand.NewSource(42)) | |
| _, err := upl.UploadWithContext(ctx, &s3manager.UploadInput{ | |
| ACL: aws.String(s3.ObjectCannedACLPublicRead), | |
| Bucket: &bucket, | |
| Key: &key, | |
| Body: io.LimitReader(rnd, size), | |
| }) | |
| return err | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment