When I tried to download AWS CLI and calculate the checksum with Go, I faced a weird issue. https://github.com/aquaproj/aqua/issues/2467

bug1.go

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"

	"github.com/codingsince1985/checksum"
)

func main() {
	if err := core(); err != nil {
		log.Fatal(err)
	}
}

func core() error {
	rc, err := download("http://awscli.amazonaws.com/AWSCLIV2-2.13.35.pkg")
	if err != nil {
		return err
	}
	defer rc.Close()
	s, err := checksum.SHA256sumReader(rc)
	if err != nil {
		return err
	}
	fmt.Println(s) // The value is changed every time
	return nil
}

func download(u string) (io.ReadCloser, error) {
	client := http.DefaultClient
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, fmt.Errorf("create a http request: %w", err)
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("send http request: %w", err)
	}
	if resp.StatusCode >= http.StatusBadRequest {
		return resp.Body, errors.New("status code >= 400")
	}
	log.Println("status code:", resp.StatusCode) // 200
	return resp.Body, nil
}

bug2.go

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/codingsince1985/checksum"
)

func main() {
	if err := core(); err != nil {
		log.Fatal(err)
	}
}

func core() error {
	rc, err := download("http://awscli.amazonaws.com/AWSCLIV2-2.13.35.pkg")
	if err != nil {
		return err
	}
	defer rc.Close()
	f, err := os.CreateTemp("", "")
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
		if err := os.Remove(f.Name()); err != nil {
			log.Println(err)
		}
	}()
	if _, err := io.Copy(f, rc); err != nil {
		return err
	}

	s, err := checksum.SHA256sumReader(f)
	if err != nil {
		return err
	}
	fmt.Println(s) // The value is unchanged but wrong.
	return nil
}

func download(u string) (io.ReadCloser, error) {
	client := http.DefaultClient
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, fmt.Errorf("create a http request: %w", err)
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("send http request: %w", err)
	}
	if resp.StatusCode >= http.StatusBadRequest {
		return resp.Body, errors.New("status code >= 400")
	}
	log.Println("status code:", resp.StatusCode)
	return resp.Body, nil
}

good1.go

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/codingsince1985/checksum"
)

func main() {
	if err := core(); err != nil {
		log.Fatal(err)
	}
}

func core() error {
	path, err := downloadAndCopy()
	if err != nil {
		return err
	}
	defer func() {
		if err := os.Remove(path); err != nil {
			log.Println(err)
		}
	}()

	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	s, err := checksum.SHA256sumReader(f)
	if err != nil {
		return err
	}
	fmt.Println(s)
	return nil
}

func downloadAndCopy() (string, error) {
	rc, err := download("http://awscli.amazonaws.com/AWSCLIV2-2.13.35.pkg")
	if err != nil {
		return "", err
	}
	defer rc.Close()
	f, err := os.CreateTemp("", "")
	if err != nil {
		return "", err
	}
	defer func() {
		f.Close()
	}()
	if _, err := io.Copy(f, rc); err != nil {
		return "", err
	}
	return f.Name(), nil
}

func download(u string) (io.ReadCloser, error) {
	client := http.DefaultClient
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return nil, fmt.Errorf("create a http request: %w", err)
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("send http request: %w", err)
	}
	if resp.StatusCode >= http.StatusBadRequest {
		return resp.Body, errors.New("status code >= 400")
	}
	log.Println("status code:", resp.StatusCode)
	return resp.Body, nil
}

bug1.go

The result is changed every time.

$ go run bug1.go
2023/11/15 15:20:47 status code: 200
6a6f6ea93ac898f8a42c1ad32a610ac20c699dce6f49dbc36160e9ad61040dfd

$ go run bug1.go
2023/11/15 15:20:50 status code: 200
6a28ce96226289451be891e58b3afacc320c4be12cd429dc694867b38c3bc8e8

bug2.go

The result is unchanged but is wrong.

$ go run bug2.go
2023/11/15 15:21:53 status code: 200
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

$ go run bug2.go
2023/11/15 15:21:55 status code: 200
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

good1.go

The result is correct

$ go run good1.go 
2023/11/15 15:22:52 status code: 200
36d16a6156150b3940cd7e750fbd7b9c529ee93c8e0eb594283f19a197de5c42

$ go run good1.go
2023/11/15 15:22:56 status code: 200
36d16a6156150b3940cd7e750fbd7b9c529ee93c8e0eb594283f19a197de5c42

Information

• Downloaded file: AWS CLI: http://awscli.amazonaws.com/AWSCLIV2-2.13.35.pkg
• Using a third party library github.com/codingsince1985/checksum https://pkg.go.dev/github.com/codingsince1985/checksum#SHA256sumReader

$ go version
go version go1.21.4 darwin/arm64

go.mod

module foo

go 1.21.4

require github.com/codingsince1985/checksum v1.3.0

require (
	golang.org/x/crypto v0.6.0 // indirect
	golang.org/x/sys v0.5.0 // indirect
)

I tried some URLs, but I couldn't reproduce the issue with URL other than the URL of AWS CLI.

• https://github.com/1xyz/pryrite/releases/download/0.10.20/pryrite-darwin-amd64-0.10.20.zip
• https://github.com/suzuki-shunsuke/mkghtag/releases/download/v0.1.1/mkghtag_darwin_arm64.tar.gz
• https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.7.2/cloud-sql-proxy.darwin.arm64