Skip to content

Instantly share code, notes, and snippets.

@vlastv
Created February 8, 2021 09:39
Show Gist options
  • Save vlastv/add9f6951b5fa0c6b88cb9a664573f33 to your computer and use it in GitHub Desktop.
Save vlastv/add9f6951b5fa0c6b88cb9a664573f33 to your computer and use it in GitHub Desktop.
Sample upload S3
package main
import (
"bytes"
"context"
"crypto/rand"
"fmt"
"log"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func main() {
bucket := aws.String("test")
size := 1024 * 1024
cli := s3.New(s3.Options{
EndpointResolver: s3.EndpointResolverFromURL("https://s3.selcdn.ru"),
Region: "ru-1a",
UsePathStyle: true,
Credentials: credentials.NewStaticCredentialsProvider("135815_test", "hH[4m*#,5s", ""),
Retryer: aws.NopRetryer{},
})
b := make([]byte, size)
i := 0
for range time.Tick(1 * time.Second) {
i++
key := fmt.Sprintf("%d.txt", i)
fmt.Println(key)
rand.Read(b)
for {
ctx, cancel := context.WithTimeout(context.TODO(), 1*time.Minute)
_, err := cli.PutObject(ctx, &s3.PutObjectInput{
Bucket: bucket,
Key: aws.String(key),
Body: bytes.NewReader(b),
})
cancel()
if err != nil {
log.Println(err)
time.Sleep(2 * time.Minute)
}
if err == nil {
break
}
}
out, err := cli.HeadObject(context.TODO(), &s3.HeadObjectInput{
Bucket: bucket,
Key: aws.String(key),
})
if err != nil {
log.Fatalln(err)
}
if out.ContentLength != int64(size) {
log.Println(key, out.ContentLength, time.Now())
}
if i == 20 {
break
}
}
}
@vlastv
Copy link
Author

vlastv commented Feb 8, 2021

sh-5.1$ go run s3test/main.go 
1.txt
2021/02/08 12:09:12 operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , canceled, context deadline exceeded
2021/02/08 12:11:12 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 066edc63-9ffb-4891-bdf2-f26a5bc10c32, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:13:12 operation error S3: PutObject, https response error StatusCode: 409, RequestID: e129718f-0e0c-4a68-a911-2dd1dedf04ad, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:15:13 operation error S3: PutObject, https response error StatusCode: 409, RequestID: dd8411ca-287c-4a58-bdf6-51a0e683107a, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:17:13 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 1ca293e2-7c34-4c69-bdbb-447cbfc12d94, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:19:13 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 6d1ed3ed-eafa-45c8-b35a-971b264988f9, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:21:13 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 0bfd6f8d-131d-4e2c-b72b-f41ced0e072e, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2.txt
3.txt
4.txt
5.txt
6.txt
7.txt
8.txt
9.txt
10.txt
11.txt
12.txt
13.txt
14.txt
15.txt
16.txt
17.txt
18.txt
19.txt
20.txt

@vlastv
Copy link
Author

vlastv commented Feb 8, 2021

sh-5.1$ go run s3test/main.go 
1.txt
2.txt
3.txt
4.txt
5.txt
6.txt
7.txt
8.txt
9.txt
10.txt
11.txt
12.txt
13.txt
14.txt
15.txt
16.txt
17.txt
18.txt
19.txt
2021/02/08 12:50:22 operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , canceled, context deadline exceeded
2021/02/08 12:52:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: c725c56e-fe22-494a-9b87-f71ecf441327, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:54:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 416d2684-a3aa-4530-b199-b40e275052a5, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:56:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 3ea9ff0e-69fb-46b4-822a-aa102a655268, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 12:58:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: 11c4bf30-c336-43b9-8c45-7913dcbb129d, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 13:00:23 operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put "https://s3.selcdn.ru/test/19.txt?x-id=PutObject": http2: Transport: cannot retry err [stream error: stream ID 1; REFUSED_STREAM] after Request.Body was written; define Request.GetBody to avoid this error
2021/02/08 13:02:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: be58c7fa-b98a-43f1-b756-2863203cb60e, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
2021/02/08 13:04:23 operation error S3: PutObject, https response error StatusCode: 409, RequestID: db9cac82-9e1d-4f9f-b2a2-8464eab60a54, HostID: , api error OperationAborted: A conflicting conditional operation is currently in progress against this resource. Try again.
20.txt

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment