Skip to content

Instantly share code, notes, and snippets.

@suzuki-shunsuke
Created March 14, 2024 11:46
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save suzuki-shunsuke/042f15cb1e3fa7d2b30366a74c09a2a2 to your computer and use it in GitHub Desktop.
Save suzuki-shunsuke/042f15cb1e3fa7d2b30366a74c09a2a2 to your computer and use it in GitHub Desktop.
A HTTP Request to crates.io with Go causes 403 (violation of crates.io crawler policy)
package main
import (
"fmt"
"io"
"log"
"net/http"
"os"
)
func main() {
if err := core(); err != nil {
log.Fatal(err)
}
}
func core() error {
// curl -o /dev/null -w '%{http_code}\n' -s https://crates.io/api/v1/crates/skim/versions
// 200
req, err := http.NewRequest("GET", "https://crates.io/api/v1/crates/skim/versions", nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode) // 403
// {"errors":[{"detail":"We are unable to process your request at this time. This usually means that you are in violation of our crawler policy (https://crates.io/policies#crawlers). Please open an issue at https://github.com/rust-lang/crates.io or email help@crates.io and provide the request id xxx"}]}
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
return err
}
return nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment