Skip to content

Instantly share code, notes, and snippets.

@tscolari
Last active April 27, 2020 09:45
Show Gist options
  • Save tscolari/7f6a95285cf51201663fd164433855f8 to your computer and use it in GitHub Desktop.
Save tscolari/7f6a95285cf51201663fd164433855f8 to your computer and use it in GitHub Desktop.
S3 Upload GOLANG different behaviours

It seems that if you use a file reader as the reader of the request (even if you wrap it into another object), golang's request to S3 will always end with a 501 Not Implemented error response. If you load the same file in memory (e.g. a buffer) this doesn't happen and it works.

go run main.go buffer ./test-file <some presigned string>
works
go run main.go reader ./test-file <some presigned string>
invalid response code: 501 Not Implemented
<Error><Code>NotImplemented</Code><Message>A header you provided implies functionality that is not implemented</Message><Header>Transfer-Encoding</Header><RequestId>F2D54740EA621AA8</RequestId><HostId>uVkv+diBWzEO9xaCIXTj45mm5b6HhIkrcwmBSJxIoJ1XzJeOv+fbEphs58JK9PkTIriENefs41w=</HostId></Error>
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strconv"
)
func main() {
if len(os.Args) != 4 {
fmt.Fprintln(os.Stderr, "ARGS: <mode(buffer|reader)> <filepath> <pre-signedurl>")
os.Exit(1)
}
file, err := os.Open(os.Args[2])
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
stat, err := file.Stat()
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
var request *http.Request
switch os.Args[1] {
case "buffer":
request, err = requestFromBuffer(os.Args[3], file)
case "reader":
request, err = requestFromReader(os.Args[3], file)
default:
err = fmt.Errorf("invalid mode: %s", os.Args[1])
}
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
request.Header.Add("Content-Length", strconv.FormatInt(stat.Size(), 10))
if err := upload(request); err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(1)
}
fmt.Println("done")
}
func upload(req *http.Request) error {
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("invalid response code: %s | body: %s", resp.Status, string(body))
}
return nil
}
func requestFromReader(url string, r io.Reader) (*http.Request, error) {
request, err := http.NewRequest("PUT", url, r)
if err != nil {
return nil, fmt.Errorf("failed to upload: %w", err)
}
return request, nil
}
func requestFromBuffer(url string, r io.Reader) (*http.Request, error) {
body := bytes.NewBuffer([]byte{})
// copy the file content to memory
if _, err := io.Copy(body, r); err != nil {
return nil, fmt.Errorf("failed to copy to buffer: %w", err)
}
request, err := http.NewRequest("PUT", url, body)
if err != nil {
return nil, fmt.Errorf("failed to upload: %w", err)
}
return request, nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment