Skip to content

Instantly share code, notes, and snippets.

@aojea

aojea/cliengo_retry_connection_reset_peer.go

Last active January 22, 2024 07:43
Show Gist options
  • Save aojea/31ab71c894c15f46a567c5e8aa235a17 to your computer and use it in GitHub Desktop.
Save aojea/31ab71c894c15f46a567c5e8aa235a17 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
cliengo_retry_connection_reset_peer.go
package main
import (
"context"
"flag"
"fmt"
"net"
"os"
"time"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
)
func main() {
klog.InitFlags(nil)
flag.Set("v", "7")
flag.Parse()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
panic(err)
}
defer ln.Close()
go func() {
for {
// Listen for an incoming connection.
conn, err := ln.Accept()
if err != nil {
fmt.Println("Error accepting: ", err.Error())
os.Exit(1)
}
// reset connection
conn.Close()
}
}()
config := &rest.Config{
Host: ln.Addr().String(),
// large timeout, otherwise the broken connection will be cleaned by it
Timeout: wait.ForeverTestTimeout,
// These fields are required to create a REST client.
ContentConfig: rest.ContentConfig{
GroupVersion: &schema.GroupVersion{},
NegotiatedSerializer: &serializer.CodecFactory{},
},
}
client, err := rest.RESTClientFor(config)
if err != nil {
panic("failed to create REST client")
}
now := time.Now()
_, err = client.Get().AbsPath("/").DoRaw(context.TODO())
if err != nil {
fmt.Println("----------------------------")
fmt.Printf("duration error: %v \n err: %v\n", time.Since(now), err)
return
}
fmt.Println("----------------------------")
fmt.Printf("unexpected duration: %v\n\n", time.Since(now))
}
Raw
clientgo_retry_50x_retry_after.go
package main
import (
"context"
"flag"
"fmt"
"net/http"
"net/http/httptest"
"time"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
)
func main() {
klog.InitFlags(nil)
flag.Set("v", "7")
flag.Parse()
ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
//fmt.Fprintf(w, "Hello, %s", r.Proto)
w.Header().Set("Retry-After", "1")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("500 - Something bad happened!"))
}))
ts.EnableHTTP2 = false
ts.StartTLS()
defer ts.Close()
transport, ok := ts.Client().Transport.(*http.Transport)
if !ok {
panic("failed to assert *http.Transport")
}
config := &rest.Config{
Host: ts.URL,
Transport: transport,
// large timeout, otherwise the broken connection will be cleaned by it
Timeout: wait.ForeverTestTimeout,
// These fields are required to create a REST client.
ContentConfig: rest.ContentConfig{
GroupVersion: &schema.GroupVersion{},
NegotiatedSerializer: &serializer.CodecFactory{},
},
}
client, err := rest.RESTClientFor(config)
if err != nil {
panic("failed to create REST client")
}
now := time.Now()
_, err = client.Get().AbsPath("/").DoRaw(context.TODO())
if err != nil {
fmt.Println("----------------------------")
fmt.Printf("duration error: %v \n err: %v\n", time.Since(now), err)
return
}
fmt.Println("----------------------------")
fmt.Printf("unexpected duration: %v", time.Since(now))
}
Raw
results.md

Client-go retry

client-go is the client used to communicate with a Kubernetes apiserver

client-go implements some retry logic for requests it considers retryable, generally:

  • no write requests (PUT,POST) , a GET is retryable
  • transient network errors: connection reset by peer or EOF
  • retryable http errors: 50x code with a Retry-After header

Retry an 50x with Retry-After

$ go run clientgo_retry.go 
I0506 09:22:52.057536   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:52.057628   18842 round_trippers.go:469] Request Headers:
I0506 09:22:52.057647   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:52.060300   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 2 milliseconds
I0506 09:22:53.061235   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 1 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:53.061325   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:53.061339   18842 round_trippers.go:469] Request Headers:
I0506 09:22:53.061357   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:53.061821   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:54.062547   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 2 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:54.062636   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:54.062651   18842 round_trippers.go:469] Request Headers:
I0506 09:22:54.062669   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:54.063019   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:55.063223   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 3 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:55.063314   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:55.063327   18842 round_trippers.go:469] Request Headers:
I0506 09:22:55.063345   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:55.063781   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:56.063987   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 4 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:56.064065   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:56.064079   18842 round_trippers.go:469] Request Headers:
I0506 09:22:56.064098   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:56.064460   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:57.064664   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 5 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:57.064735   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:57.064750   18842 round_trippers.go:469] Request Headers:
I0506 09:22:57.064767   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:57.065102   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:58.065345   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 6 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:58.065462   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:58.065476   18842 round_trippers.go:469] Request Headers:
I0506 09:22:58.065495   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:58.065924   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:22:59.066563   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 7 to https://127.0.0.1:33267/?timeout=30s
I0506 09:22:59.066647   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:22:59.066660   18842 round_trippers.go:469] Request Headers:
I0506 09:22:59.066678   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:22:59.067151   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:23:00.067424   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 8 to https://127.0.0.1:33267/?timeout=30s
I0506 09:23:00.067535   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:23:00.067549   18842 round_trippers.go:469] Request Headers:
I0506 09:23:00.067568   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:23:00.068015   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:23:01.068607   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 9 to https://127.0.0.1:33267/?timeout=30s
I0506 09:23:01.068731   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:23:01.068746   18842 round_trippers.go:469] Request Headers:
I0506 09:23:01.068769   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:23:01.069222   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
I0506 09:23:02.070220   18842 with_retry.go:241] Got a Retry-After 1s response for attempt 10 to https://127.0.0.1:33267/?timeout=30s
I0506 09:23:02.070295   18842 round_trippers.go:463] GET https://127.0.0.1:33267/?timeout=30s
I0506 09:23:02.070308   18842 round_trippers.go:469] Request Headers:
I0506 09:23:02.070324   18842 round_trippers.go:473]     Accept: application/json, */*
I0506 09:23:02.070707   18842 round_trippers.go:574] Response Status: 500 Internal Server Error in 0 milliseconds
duration error: 10.013440484s

The request blocks by 10 seconds by default, it retries 10 times by default and the Retry-After is 1 second

Retry on connection reset by peer

go run clientgo_retry.go 
I0506 09:34:15.859757   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:15.859855   25831 round_trippers.go:469] Request Headers:
I0506 09:34:15.859876   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:15.860613   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:16.860930   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 1 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:16.861031   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:16.861045   25831 round_trippers.go:469] Request Headers:
I0506 09:34:16.861062   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:16.861624   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:17.861909   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 2 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:17.862004   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:17.862017   25831 round_trippers.go:469] Request Headers:
I0506 09:34:17.862032   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:17.862559   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:18.863667   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 3 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:18.863842   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:18.863910   25831 round_trippers.go:469] Request Headers:
I0506 09:34:18.863944   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:18.864841   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:19.867564   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 4 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:19.867643   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:19.867656   25831 round_trippers.go:469] Request Headers:
I0506 09:34:19.867674   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:19.868137   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:20.868611   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 5 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:20.868699   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:20.868771   25831 round_trippers.go:469] Request Headers:
I0506 09:34:20.868806   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:20.869345   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:21.870674   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 6 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:21.870825   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:21.870892   25831 round_trippers.go:469] Request Headers:
I0506 09:34:21.870925   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:21.871820   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:22.873518   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 7 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:22.873838   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:22.874004   25831 round_trippers.go:469] Request Headers:
I0506 09:34:22.874092   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:22.874945   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:23.875557   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 8 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:23.875636   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:23.875650   25831 round_trippers.go:469] Request Headers:
I0506 09:34:23.875671   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:23.876555   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:24.877532   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 9 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:24.877589   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:24.877595   25831 round_trippers.go:469] Request Headers:
I0506 09:34:24.877605   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:24.878053   25831 round_trippers.go:574] Response Status:  in 0 milliseconds
I0506 09:34:25.879009   25831 with_retry.go:241] Got a Retry-After 1s response for attempt 10 to http://127.0.0.1:38541/?timeout=30s
I0506 09:34:25.879780   25831 round_trippers.go:463] GET http://127.0.0.1:38541/?timeout=30s
I0506 09:34:25.880638   25831 round_trippers.go:469] Request Headers:
I0506 09:34:25.880771   25831 round_trippers.go:473]     Accept: application/json, */*
I0506 09:34:25.884899   25831 round_trippers.go:574] Response Status:  in 4 milliseconds
duration error: 10.02540544s 
 err: Get "http://127.0.0.1:38541/?timeout=30s": read tcp 127.0.0.1:33656->127.0.0.1:38541: read: connection reset by peer - error from a previous attempt: read tcp 127.0.0.1:33654->127.0.0.1:38541: read: connection reset by peer
Error accepting:  accept tcp 127.0.0.1:38541: use of closed network connection

The request blocks by 10 seconds by default, it retries 10 times by default and the logic inserts a synthetic Retry-After of 1 second, that is what you can see in the log trace

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment