-
-
Save pharaujo/eb2d4697e8352883ce8c050f8666003c to your computer and use it in GitHub Desktop.
Go routine dump
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine 10853071 [running]: | |
| runtime/pprof.writeGoroutineStacks({0x7f0a8badacc8, 0xc042401a40}) | |
| /usr/local/go/src/runtime/pprof/pprof.go:693 +0x70 | |
| runtime/pprof.writeGoroutine({0x7f0a8badacc8, 0xc042401a40}, 0x0) | |
| /usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b | |
| runtime/pprof.(*Profile).WriteTo(0x2974720, {0x7f0a8badacc8, 0xc042401a40}, 0xc) | |
| /usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b | |
| net/http/pprof.handler.ServeHTTP({0xc099f4ec3d, 0x4064c5}, {0x7f0a8bac7e40, 0xc042401a40}, 0x8) | |
| /usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a | |
| net/http/pprof.Index({0x7f0a8bac7e40, 0xc042401a40}, 0xc042255700) | |
| /usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e | |
| github.com/prometheus/prometheus/web.serveDebug({0x7f0a8bac7e40, 0xc042401a40}, 0xc042255700) | |
| /app/web/web.go:532 +0x2c5 | |
| net/http.HandlerFunc.ServeHTTP(0x7f0a8bac7e40, {0x7f0a8bac7e40, 0xc042401a40}, 0xc000100000) | |
| /usr/local/go/src/net/http/server.go:2047 +0x2f | |
| github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x7f0a8bac7e40, 0xc0424019f0}, 0xc042255700) | |
| /go/pkg/mod/github.com/prometheus/client_golang@v1.11.0/prometheus/promhttp/instrument_server.go:198 +0xa3 | |
| net/http.HandlerFunc.ServeHTTP(0x354ea00, {0x7f0a8bac7e40, 0xc0424019f0}, 0xc096ed4990) | |
| /usr/local/go/src/net/http/server.go:2047 +0x2f | |
| github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2({0x7f0a8bac7e40, 0xc0424019f0}, 0xc042255700) | |
| /go/pkg/mod/github.com/prometheus/client_golang@v1.11.0/prometheus/promhttp/instrument_server.go:76 +0xa2 | |
| net/http.HandlerFunc.ServeHTTP(0x35899d0, {0x7f0a8bac7e40, 0xc0424019f0}, 0x0) | |
| /usr/local/go/src/net/http/server.go:2047 +0x2f | |
| github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x35899d0, 0xc0424019a0}, 0xc042255700) | |
| /go/pkg/mod/github.com/prometheus/client_golang@v1.11.0/prometheus/promhttp/instrument_server.go:101 +0x92 | |
| github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1({0x35899d0, 0xc0424019a0}, 0xc042255600) | |
| /app/web/web.go:1173 +0x290 | |
| github.com/prometheus/common/route.(*Router).handle.func1({0x35899d0, 0xc0424019a0}, 0xc042255500, {0xc042269ec0, 0x1, 0xc000100000}) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/route/route.go:83 +0x2ae | |
| github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc00112a2a0, {0x35899d0, 0xc0424019a0}, 0xc042255500) | |
| /go/pkg/mod/github.com/julienschmidt/httprouter@v1.3.0/router.go:387 +0x84b | |
| github.com/prometheus/common/route.(*Router).ServeHTTP(0x7f09ed8a7c10, {0x35899d0, 0xc0424019a0}, 0x7f0ab2b3f108) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/route/route.go:126 +0x26 | |
| net/http.(*ServeMux).ServeHTTP(0x40d187, {0x35899d0, 0xc0424019a0}, 0xc042255500) | |
| /usr/local/go/src/net/http/server.go:2425 +0x149 | |
| github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5({0x35884d0, 0xc04e5157a0}, 0xc042255400) | |
| /go/pkg/mod/github.com/opentracing-contrib/go-stdlib@v1.0.0/nethttp/server.go:154 +0x62d | |
| net/http.HandlerFunc.ServeHTTP(0x0, {0x35884d0, 0xc04e5157a0}, 0xc001ef87e0) | |
| /usr/local/go/src/net/http/server.go:2047 +0x2f | |
| github.com/prometheus/prometheus/web.withStackTracer.func1({0x35884d0, 0xc04e5157a0}, 0xc06899fe00) | |
| /app/web/web.go:110 +0x97 | |
| net/http.HandlerFunc.ServeHTTP(0x7fff74b6316a, {0x35884d0, 0xc04e5157a0}, 0x90) | |
| /usr/local/go/src/net/http/server.go:2047 +0x2f | |
| github.com/prometheus/exporter-toolkit/web.(*webHandler).ServeHTTP(0xc001093c80, {0x35884d0, 0xc04e5157a0}, 0x4668ce) | |
| /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.7.0/web/handler.go:101 +0x852 | |
| net/http.serverHandler.ServeHTTP({0xc096ed4780}, {0x35884d0, 0xc04e5157a0}, 0xc042255400) | |
| /usr/local/go/src/net/http/server.go:2879 +0x43b | |
| net/http.(*conn).serve(0xc04241b400, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1930 +0xb08 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 1 [chan receive, 2630 minutes]: | |
| github.com/oklog/run.(*Group).Run(0xc0007f7ac8) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x7c | |
| main.main() | |
| /app/cmd/prometheus/main.go:1063 +0x8bf3 | |
| goroutine 16 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc000128080) | |
| /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9 | |
| created by go.opencensus.io/stats/view.init.0 | |
| /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92 | |
| goroutine 3763 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783a00, 0xc0169bc880) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 60 [syscall, 2630 minutes]: | |
| os/signal.signal_recv() | |
| /usr/local/go/src/runtime/sigqueue.go:169 +0x98 | |
| os/signal.loop() | |
| /usr/local/go/src/os/signal/signal_unix.go:24 +0x19 | |
| created by os/signal.Notify.func1.1 | |
| /usr/local/go/src/os/signal/signal.go:151 +0x2c | |
| goroutine 43 [select]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).sender(0xc001057d60) | |
| /app/discovery/legacymanager/manager.go:235 +0xdd | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run | |
| /app/discovery/legacymanager/manager.go:142 +0x5f | |
| goroutine 88 [select]: | |
| github.com/prometheus/prometheus/util/logging.(*Deduper).run(0xc0010920c0) | |
| /app/util/logging/dedupe.go:75 +0xee | |
| created by github.com/prometheus/prometheus/util/logging.Dedupe | |
| /app/util/logging/dedupe.go:61 +0x10f | |
| goroutine 95 [select]: | |
| github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run(0xc001057c20) | |
| /app/storage/remote/write.go:107 +0xdd | |
| created by github.com/prometheus/prometheus/storage/remote.NewWriteStorage | |
| /app/storage/remote/write.go:99 +0x485 | |
| goroutine 61 [select, 2630 minutes]: | |
| main.main.func6() | |
| /app/cmd/prometheus/main.go:774 +0xb1 | |
| github.com/oklog/run.(*Group).Run.func1({0xc00014aec0, 0xc0006f0120}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 62 [chan receive, 2630 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run(0xc001057cc0) | |
| /app/discovery/legacymanager/manager.go:143 +0x7c | |
| main.main.func8() | |
| /app/cmd/prometheus/main.go:794 +0x38 | |
| github.com/oklog/run.(*Group).Run.func1({0xc001040000, 0xc00052a380}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 63 [chan receive, 2630 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run(0xc001057d60) | |
| /app/discovery/legacymanager/manager.go:143 +0x7c | |
| main.main.func10() | |
| /app/cmd/prometheus/main.go:808 +0x38 | |
| github.com/oklog/run.(*Group).Run.func1({0xc001040030, 0xc00052a420}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 64 [select, 1 minutes]: | |
| github.com/prometheus/prometheus/scrape.(*Manager).Run(0xc0006ccf60, 0xc0010ac000) | |
| /app/scrape/manager.go:150 +0xc5 | |
| main.main.func12() | |
| /app/cmd/prometheus/main.go:828 +0x85 | |
| github.com/oklog/run.(*Group).Run.func1({0xc00014af80, 0xc00052a460}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 273 [select, 2624 minutes]: | |
| main.main.func14() | |
| /app/cmd/prometheus/main.go:853 +0x145 | |
| github.com/oklog/run.(*Group).Run.func1({0xc0005201e0, 0xc0006f01a0}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 274 [chan receive, 2624 minutes]: | |
| main.main.func16() | |
| /app/cmd/prometheus/main.go:898 +0x2d8 | |
| github.com/oklog/run.(*Group).Run.func1({0xc000520480, 0xc0006f01b0}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 275 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/rules.(*Manager).Run(...) | |
| /app/rules/manager.go:933 | |
| main.main.func18() | |
| /app/cmd/prometheus/main.go:911 +0x4b | |
| github.com/oklog/run.(*Group).Run.func1({0xc00000c048, 0xc0006f0850}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 276 [chan receive, 2624 minutes]: | |
| main.main.func20() | |
| /app/cmd/prometheus/main.go:963 +0xb56 | |
| github.com/oklog/run.(*Group).Run.func1({0xc00009a050, 0xc001040060}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 277 [select, 2630 minutes]: | |
| github.com/prometheus/prometheus/web.(*Handler).Run(0xc001063b80, {0x35a0c90, 0xc0010921c0}, {0x357dbb0, 0xc001161410}, {0x7fff74b6316a, 0x2a}) | |
| /app/web/web.go:626 +0x7f9 | |
| main.main.func24() | |
| /app/cmd/prometheus/main.go:1031 +0x38 | |
| github.com/oklog/run.(*Group).Run.func1({0xc00014afc0, 0xc0006f0860}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 278 [select]: | |
| github.com/prometheus/prometheus/notifier.(*Manager).Run(0xc00107a600, 0xc0010ac0c0) | |
| /app/notifier/notifier.go:307 +0xc5 | |
| main.main.func26() | |
| /app/cmd/prometheus/main.go:1054 +0x85 | |
| github.com/oklog/run.(*Group).Run.func1({0xc00014b000, 0xc0006f0900}) | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x2f | |
| created by github.com/oklog/run.(*Group).Run | |
| /go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0x22f | |
| goroutine 267 [select]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).sender(0xc001057cc0) | |
| /app/discovery/legacymanager/manager.go:235 +0xdd | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).Run | |
| /app/discovery/legacymanager/manager.go:142 +0x5f | |
| goroutine 383 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd60218, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00112c500, 0x2, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Accept(0xc00112c500) | |
| /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c | |
| net.(*netFD).accept(0xc00112c500) | |
| /usr/local/go/src/net/fd_unix.go:173 +0x35 | |
| net.(*TCPListener).accept(0xc0011613c8) | |
| /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 | |
| net.(*TCPListener).Accept(0xc0011613c8) | |
| /usr/local/go/src/net/tcpsock.go:262 +0x3d | |
| golang.org/x/net/netutil.(*limitListener).Accept(0xc000d9c7b0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/netutil/listen.go:48 +0x3e | |
| github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc001161410) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/listener_wrapper.go:100 +0x6c | |
| net/http.(*Server).Serve(0xc0001780e0, {0x357dbb0, 0xc001161410}) | |
| /usr/local/go/src/net/http/server.go:3002 +0x394 | |
| github.com/prometheus/exporter-toolkit/web.Serve({0x357dbb0, 0xc001161410}, 0xc0001780e0, {0x7fff74b6316a, 0x2a}, {0x3547580, 0xc0010596d0}) | |
| /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.7.0/web/tls_config.go:232 +0x517 | |
| github.com/prometheus/prometheus/web.(*Handler).Run.func2() | |
| /app/web/web.go:623 +0x45 | |
| created by github.com/prometheus/prometheus/web.(*Handler).Run | |
| /app/web/web.go:622 +0x779 | |
| goroutine 390 [select, 8 minutes]: | |
| github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc0001a2510) | |
| /app/tsdb/wal/wal.go:332 +0xaa | |
| created by github.com/prometheus/prometheus/tsdb/wal.NewSize | |
| /app/tsdb/wal/wal.go:301 +0x2cf | |
| goroutine 91219 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d16420, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0e10a0580, 0xc009cda000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0e10a0580, {0xc009cda000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0e10a0580, {0xc009cda000, 0xc032f58c80, 0xc01bb9ef68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc047f8cb90, {0xc009cda000, 0xc03d82bd40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04a2b2fc0, {0xc009cda000, 0xc0e1e7fe00, 0xc01bb9ed30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc08e8fc840) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc08e8fc840, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04a2b2fc0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3303 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc010575190, 0x1e9) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc012c85c80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc010243280, {0x35a0c90, 0xc0135d1940}, 0xc017d2d540) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3765 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783c70, 0xc0169bc400) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10606407 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730768, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b2ebec00, 0xc024540000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b2ebec00, {0xc024540000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b2ebec00, {0xc024540000, 0x3ccb, 0xc0afbd7d40}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01076b928, {0xc024540000, 0xc024540000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0229f5c98, {0xc024540000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0b0b39078, {0x3544be0, 0xc0229f5c98}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0b0b38e00, {0x7f09e11af0a0, 0xc0a5d57ac0}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0b0b38e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0b0b38e00, {0xc0b19b5000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0b1996f00, {0xc0014da3c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0b1996f00}, {0xc0014da3c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0014da3c0, 0x9, 0xc0011d2ae0}, {0x35449e0, 0xc0b1996f00}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014da380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01cb58f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a7204300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3737 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f5ba0, 0xc016f97900) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2449 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315e30, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2450 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 10599776 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04a132680, 0xc04cc34420) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852553 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733cf8, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0f1f79300, 0xc0d7689740, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc0f1f79300, {0x35a0d00, 0xc020bdfb60}, {0xc087a072f8, 0x40ce34}, {0x354bea0, 0xc0b294d180}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc0f1f79300, {0x35a0d00, 0xc020bdfb60}, {0x35c3098, 0x0}, {0x35c3098, 0xc0d76896e0}, 0xc087a074e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0d76896e0}, 0xc0152ff360, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc0f1f79280, {0x35a0d00, 0xc020bdfb60}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc020bdfb60, {0x35a0d00, 0xc020bdfb60}, 0xc01c7a1610, 0xc003b7c588) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc0f1f79280, {0x35a0d00, 0xc020bdfb60}, {0x357b3f8, 0xc0d76896e0}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc0f1f79280, {0x35a0d00, 0xc020bdfb60}, {0xc0256faa90, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc01978eea0, {0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0xc0256faa70}, {0xc0201403f0, 0xc068092370}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0x3}, {0xc0201403f0, 0x12}, {0xc017c69590, 0x0}, 0xc01978aec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0x3}, {0xc0201403f0, 0x12}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc020bdfb60, {0x35a0d00, 0xc020bdfb60}, {0x2c655e4, 0x0}, {0xc0201403f0, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f53900, {0x35a0d00, 0xc020bdfb60}, {{}, 0x0, {0xc035f47b80, 0x5}, {0xc0201403f0, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x6ce8c6, 0xc10197ea50) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 4302 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15b10, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc030113800, 0xc034e34000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc030113800, {0xc034e34000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc030113800, {0xc034e34000, 0xc033082f00, 0xc0148cef68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c8768e8, {0xc034e34000, 0xc0136df1e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05eeebe60, {0xc034e34000, 0xc03e47f800, 0xc0148ced30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc034712cc0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc034712cc0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05eeebe60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 91349 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01b2752b0, 0xc01802f100) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2370 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 132747 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029909450, 0xc111ea0fc0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3714 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4340, 0xc000dcc840) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3599 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ef450, 0xc000dcc900) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2369 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0123159d0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10850277 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc08bc75248, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc08bc75238) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08bc75230, {0xc0596b8600, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc0596b8600, 0xc00ccc4560, 0xc00ccc44f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05be39900) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05be39900) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05be39900, {0x2748340, 0xc031952588}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc07d8dd020, {0xc0a5e82400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00e8f24b0, 0xc0137892c0, {0x357ac00, 0xc06f2aec80}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0ba3c69a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc06f2aec40) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 4588 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b5390, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04ea27800, 0xc04fefc000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc04ea27800, {0xc04fefc000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc04ea27800, {0xc04fefc000, 0xc032f59b80, 0xc020814f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01d81f680, {0xc04fefc000, 0xc02407e340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc032f24d80, {0xc04fefc000, 0xc0357342a0, 0xc020814d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04edf8960) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04edf8960, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc032f24d80) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2436 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d420, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0c5003100}, 0xc003b37ce8, 0xc013ab4600, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d420, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31ef0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e05780, {0x354b360, 0xc012c31e50}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d420, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 93677 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5fd90, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0df167d00, 0xc0df200000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0df167d00, {0xc0df200000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0df167d00, {0xc0df200000, 0xc032f59900, 0xc020e13f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc036eaffc8, {0xc0df200000, 0xc0f770dd40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0929dc7e0, {0xc0df200000, 0xc0dd17d920, 0xc020e13d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0de9716e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0de9716e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0929dc7e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 762 [select]: | |
| github.com/prometheus/prometheus/tsdb.(*DB).run(0xc0005564e0) | |
| /app/tsdb/db.go:798 +0x18e | |
| created by github.com/prometheus/prometheus/tsdb.open | |
| /app/tsdb/db.go:748 +0xc16 | |
| goroutine 3682 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c7820d0, 0xc10194c0c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2176 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776720) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 67003 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc08f73f040, 0xc08e827780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852906 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734268, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc025094e00, 0xc0dd23e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc025094e00, {0xc0dd23e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc025094e00, {0xc0dd23e000, 0xc032f59180, 0xc003b84f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc06c20e7f0, {0xc0dd23e000, 0xc02efd7040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0162eb560, {0xc0dd23e000, 0xc0e0cc8c60, 0xc003b84d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc03973bc20) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc03973bc20, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0162eb560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10835469 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0094d9248, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0144a1ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0094d9230, {0xc00b250601, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc00b250601, 0xc0a6e14800, 0xc0144a1cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc098e88140) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc098e88140) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc098e88140, {0x2748340, 0xc039bd0a80}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08dac99b0, {0xc0d4f4c800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00e3044b0, 0xc01c299320, {0x357ac00, 0xc02d7ade40}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0a7079f20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc090289f40) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 91270 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a2096b60, 0xc017ef7b00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2490 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc01606ff38, {0x354b380, 0xc02122f2f0}, 0x1, 0xc026f4eb40) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x40, 0xc01606ff88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232ec80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3788 [select]: | |
| net/http.(*Transport).getConn(0xc032f53900, 0xc02d7ad140, {{}, 0x0, {0xc045879f40, 0x5}, {0xc023544be8, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f53900, 0xc02d850400) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0877ebef0) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02d9c1c20, 0xc02d850300) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc02d850300, {0x3548b80, 0xc02d9c1c20}, {0x2bdc000, 0xc01e877601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02d9c1c50, 0xc02d850300, {0x0, 0xc09e557758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02d9c1c50, 0xc02d850300) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01e4244c0, {0x35a0d00, 0xc02d85e000}, {0x3544a40, 0xc0877ebe00}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01cfcf040, {0xeda5786ee, 0x4d8e460, 0x4d8e460}, {0xeda5786ee, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf040, 0xc015844c00) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2409 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315c00, 0x11eaa546) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3898 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a9c0, 0xc016dffe80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2348 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315a40, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 4038 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d166d8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0226a3a00, 0xc0220d9000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0226a3a00, {0xc0220d9000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0226a3a00, {0xc0220d9000, 0xc032f523c0, 0xc01b512f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b20c830, {0xc0220d9000, 0xc01b383380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc010410900, {0xc0220d9000, 0xc036ced380, 0xc01b512d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc023525260) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc023525260, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc010410900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3876 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279ad0, 0xc0139b7b80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3600 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ef930, 0xc003a7cd80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3785 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcedd0, 0xc015845200) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2349 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 10825084 [sync.Cond.Wait, 6 minutes]: | |
| sync.runtime_notifyListWait(0xc02168f6c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc02168f6b8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc02168f6b0, {0xc09662ac00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc09662ac00, 0xc003e45d60, 0xc003e45cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc060642f00) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc060642f00) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc060642f00, {0x2748340, 0xc0216bcea0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06c343860, {0xc012049000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0216a77c0, 0xc0135d1c00, {0x357ac00, 0xc0216e85c0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05d801840) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0216e8580) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2235 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232ee80, {0x35a0c90, 0xc0135d1c00}, 0xc027b0be60) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232ee80}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 91633 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a9280dd0, 0xc039be62c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2272 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc01606df38, {0x354b380, 0xc01fde3f20}, 0x1, 0xc02749af60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x40, 0xc01606df88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc010243300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3780 [select]: | |
| net/http.(*Transport).getConn(0xc032f53900, 0xc020f3f240, {{}, 0x0, {0xc0189b8840, 0x5}, {0xc00fab7d88, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f53900, 0xc06be49800) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc016690330) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02d9c1c20, 0xc06be49700) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc06be49700, {0x3548b80, 0xc02d9c1c20}, {0x2bdc000, 0xc00fbb7601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02d9c1c50, 0xc06be49700, {0x0, 0xc09e5d3758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02d9c1c50, 0xc06be49700) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc0196b7ec0, {0x35a0d00, 0xc020bdf0e0}, {0x3544a40, 0xc016690210}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01cfce820, {0xeda5786e6, 0x4d8e460, 0x4d8e460}, {0xeda5786e6, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce820, 0xc0177d3dc0) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 1547 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc012c85c80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 3759 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee29c0, 0xc01512ec80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3760 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee2750, 0xc01512e780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10806751 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06698b0e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4108 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a5ea0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02389a400, 0xc023902000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02389a400, {0xc023902000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02389a400, {0xc023902000, 0xc032f59b80, 0xc01c7a5f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b3fc608, {0xc023902000, 0xc013c09520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc01be6ac60, {0xc023902000, 0xc038836cc0, 0xc01c7a5d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc022a956e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc022a956e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc01be6ac60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2739 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2473 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2492 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003a06738, {0x354b380, 0xc02122f320}, 0x1, 0xc026f4eba0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0xb0, 0xc003a06788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232ed00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3595 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185eeea0, 0xc018f02fc0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10762830 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0632b9b00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3771 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783930, 0xc01691a280) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3769 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e5965b0, 0xc01691a880) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3767 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783e10, 0xc0169bc000) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3862 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0203596c0, 0xc016dfe300) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2469 [chan receive, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00398b738, {0x354b380, 0xc020f6f140}, 0x1, 0xc0285aa720) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0xb0, 0xc00398b788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232fe00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2410 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2440 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe2a0, 0xc01a308960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3700 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c782410, 0xc016aac680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2447 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003a22f38, {0x354b380, 0xc020f6f0b0}, 0x1, 0xc0285aa600) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01aa54840, 0x3b9aca00, 0x0, 0xb8, 0xc003a22f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232fa00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2363 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 10825083 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc02168f680, 0xc04ca38900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc06c343350) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 93397 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bab28, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ddb96b80, 0xc0d88f9000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ddb96b80, {0xc0d88f9000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ddb96b80, {0xc0d88f9000, 0xc032f58dc0, 0xc003aebf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc037a04310, {0xc0d88f9000, 0xc0c10f44e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03d6a10e0, {0xc0d88f9000, 0xc0dd17d2c0, 0xc003aebd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0c97c39e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0c97c39e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03d6a10e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3776 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596750, 0xc014f63080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3810 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783790, 0xc014f62c80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3790 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf790, 0xc015844680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852568 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b4c50, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0204a6700, 0xc0841d2570, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc0204a6700, {0x35a0d00, 0xc0229c4cc0}, {0xc06f9cb2f8, 0x40ce34}, {0x354bea0, 0xc0dcae5c00}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc0204a6700, {0x35a0d00, 0xc0229c4cc0}, {0x35c3098, 0x0}, {0x35c3098, 0xc0841d2540}, 0xc06f9cb4e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0841d2540}, 0xc062011f20, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc0204a6680, {0x35a0d00, 0xc0229c4cc0}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc0229c4cc0, {0x35a0d00, 0xc0229c4cc0}, 0x0, 0x0) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc0204a6680, {0x35a0d00, 0xc0229c4cc0}, {0x357b3f8, 0xc0841d2540}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc0204a6680, {0x35a0d00, 0xc0229c4cc0}, {0xc0229dfdf0, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc0197b7500, {0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0xc0229dfdd0}, {0xc03e8051e8, 0xc0a2e7c580}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0x3}, {0xc03e8051e8, 0x11}, {0xc017c696e0, 0x0}, 0xc0197b2ec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0x3}, {0xc03e8051e8, 0x11}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc0229c4cc0, {0x35a0d00, 0xc0229c4cc0}, {0x2c655e4, 0x0}, {0xc03e8051e8, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f59e00, {0x35a0d00, 0xc0229c4cc0}, {{}, 0x0, {0xc01a006400, 0x5}, {0xc03e8051e8, 0x11}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x35a1588, 0xc0018c74a0) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 3787 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf2b0, 0xc015844e00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2371 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765a40, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02ccddc80}, 0xc003b19ce8, 0xc013ab4420, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765a40, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0130358b0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6940, {0x354b360, 0xc013035810}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765a40, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 4045496 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc017db1450, 0xc009d8ec00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10841058 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc01120b380, 0xc08d488a00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc05b4542c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 2533 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2467 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003a20738, {0x354b380, 0xc020f6f110}, 0x1, 0xc0285aa6c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1a382df0, 0x3b9aca00, 0x0, 0x46, 0xc003a20788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232fc80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 6199302 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0602ba900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2470 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232fe00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 93451 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0acc8dd40) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2396 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe150, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2398 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d340, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc048a17700}, 0xc105fedce8, 0xc013a3f2c0, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d340, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31db0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e05640, {0x354b360, 0xc012c31d10}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d340, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3701 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c7824e0, 0xc016aac480) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2434 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe230, 0xc01b7bc6c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3851 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020358820, 0xc016f52780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68066 [select]: | |
| net/http.(*persistConn).writeLoop(0xc09ba505a0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3764 [select]: | |
| net/http.(*Transport).getConn(0xc032f59e00, 0xc0229f8540, {{}, 0x0, {0xc01a006400, 0x5}, {0xc03e8051e8, 0x11}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f59e00, 0xc082d42e00) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0841d24b0) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02dda0c00, 0xc082d42d00) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc082d42d00, {0x3548b80, 0xc02dda0c00}, {0x2bdc000, 0xc016f1b601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02dda0c30, 0xc082d42d00, {0x0, 0xc09e6f9758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02dda0c30, 0xc082d42d00) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01c781ec0, {0x35a0d00, 0xc0229c4cc0}, {0x3544a40, 0xc0841d23f0}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01e596000, {0xeda5786e8, 0x4d8e460, 0x4d8e460}, {0xeda5786e8, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596000, 0xc0169bc600) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2423 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488a000, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01c2c1c00}, 0xc013a15ce8, 0xc013f16ea0, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488a000, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a080f0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7040, {0x354b360, 0xc013a08050}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488a000, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2527 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 68491 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc091946270, 0xc0b03baba0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 9823683 [select]: | |
| net/http.(*persistConn).writeLoop(0xc068545560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3588 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ee820, 0xc000676400) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2435 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2428 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 68737 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fede8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a1219180, 0xc05ee55000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a1219180, {0xc05ee55000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a1219180, {0xc05ee55000, 0xc032f59cc0, 0xc003a6bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc001c990f0, {0xc05ee55000, 0xc0b3112d00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08e4fa7e0, {0xc05ee55000, 0xc03cab50e0, 0xc003a6bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a6616ba0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a6616ba0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08e4fa7e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2334 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2126 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc013ffa8c0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f16d580) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 94766 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b5b0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c46d600, 0xc0d54db000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c46d600, {0xc0d54db000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c46d600, {0xc0d54db000, 0xc033082dc0, 0xc0a6066f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc000f73e78, {0xc0d54db000, 0xc0367601a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04f1938c0, {0xc0d54db000, 0xc012dd4c00, 0xc0a6066d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc08f62d6e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc08f62d6e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04f1938c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2119 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138ceee0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f828200) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2122 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01446b420, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf5e00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2503 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f400) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2427 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315d50, 0xc003a8a7a8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10833591 [sync.Cond.Wait, 5 minutes]: | |
| sync.runtime_notifyListWait(0xc07f9790c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc07f9790b8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc07f9790b0, {0xc03d321e00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc03d321e00, 0xc0045f6560, 0xc0045f64f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc082491900) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc082491900) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc082491900, {0x2748340, 0xc0824bfe30}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0824bb950, {0xc03ad90000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc08248b360, 0xc082401860, {0x357ac00, 0xc0824addc0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc082499960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0824add80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2249 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232f380, {0x35a0c90, 0xc0135d1d00}, 0xc027b0bf20) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232f380}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 4047280 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b32d0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc08a236000, 0xc08a650000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc08a236000, {0xc08a650000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc08a236000, {0xc08a650000, 0xc032c9be00, 0xc003bacf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00e134530, {0xc08a650000, 0xc0137e9520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc080f7ea20, {0xc08a650000, 0xc0218e5200, 0xc003bacd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc080ead200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc080ead200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc080f7ea20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3852 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020358a90, 0xc016f52580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2289 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776ae0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2565 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2125 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc013ffa1c0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf5500) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 10816833 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc06e1b6ac8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xac) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc06e1b6ab0, {0xc0d5334c01, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc0d5334c01, 0xc0600bde00, 0xc06f9d1cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc06ce19b80) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc06ce19b80) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc06ce19b80, {0x2748340, 0xc01dcdb770}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06e235ce0, {0xc06e20f400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc06e216960, 0xc050f1bc20, {0x357ac00, 0xc01dcd2a00}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc06e230fc0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc06e2470c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2252 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776a20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2254 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc013788300, {0x35a0c90, 0xc0135d1d80}, 0xc027b0bf80) | |
| /app/discovery/kubernetes/pod.go:103 +0x1ef | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548ea0, 0xc013788300}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 3854 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0203581a0, 0xc016f53300) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2391 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2118 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0341820e0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f828400) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 10603388 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc074fd2000, 0xc03f1db440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2455 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315dc0, 0xc003a8bfa8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2110 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138cee00, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x5f5f656d616e5f5f) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2462 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2314 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057d60, {0x35a0c90, 0xc0137892c0}, 0xc0138529b0, 0xc028765800) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2109 [select, 1 minutes]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138cec40, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x7363697274) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2641 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2248 [sync.Cond.Wait, 58 minutes]: | |
| sync.runtime_notifyListWait(0xc012b836a8, 0xc7) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc06d8a63e0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83680, 0xc013a01a40) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a2c510) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122ee10}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a2c578, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a2c510, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e0a0, 0xc000142f78) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 3690 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185efe10, 0x0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10851910 [select]: | |
| net/http.(*persistConn).writeLoop(0xc027370000) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10833610 [select, 5 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc07f979080, 0xc0822f6800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0822bce40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 2392 [select, 6 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d260, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0530f8540}, 0xc003bbbce8, 0xc01324aea0, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d260, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31c70) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e05480, {0x354b360, 0xc012c31bd0}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d260, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3816 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783520, 0xc014f21f80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3590 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ee9c0, 0xc03387f4a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2226 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83068, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e3820) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83040, 0xc011d27d20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013071170) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020725560}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0130711d8, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013071170, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0137139a0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2247 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83608, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e3640) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b835e0, 0xc013a01760) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a2c000) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122eba0}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a2c068, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a2c000, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713f40, 0xc003aa5f78) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 3583 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4f70, 0xc00536c2a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2531 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc013a1e600) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3773 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596340, 0xc014f63680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3829 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6158, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc020ba8100, 0xc020995000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc020ba8100, {0xc020995000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc020ba8100, {0xc020995000, 0xc033083400, 0xc01ac75f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01aa2c418, {0xc020995000, 0xc028be5520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04e6910e0, {0xc020995000, 0xc01e754f60, 0xc01ac75d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc01f7cd860) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc01f7cd860, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04e6910e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3784 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfced00, 0xc015845500) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2246 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83568, 0x172) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b8ca0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83540, 0xc013a01480) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a05a70) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122e930}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a05ad8, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a05a70, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e000, 0xc003aa7778) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 2636 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 3587 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ee680, 0xc01938c8c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3713 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f41a0, 0xc105fac6c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 97807 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb098, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a511fb00, 0xc0a4a0d000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a511fb00, {0xc0a4a0d000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a511fb00, {0xc0a4a0d000, 0xc032f52c80, 0xc003b2cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0551d25d0, {0xc0a4a0d000, 0xc029637380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02602b9e0, {0xc0a4a0d000, 0xc0688998c0, 0xc003b2cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a4501a40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a4501a40, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02602b9e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2456 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2457 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488ab60, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc063d26ac0}, 0xc013a13ce8, 0xc014510780, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488ab60, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a08500) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7480, {0x354b360, 0xc013a08460}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488ab60, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2411 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765dc0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc03dbf9b00}, 0xc003bb5ce8, 0xc013e7ecc0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765dc0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035e00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6d80, {0x354b360, 0xc013035d60}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765dc0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3758 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee28f0, 0xc01512ee80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4051602 [select]: | |
| net/http.(*persistConn).writeLoop(0xc086682360) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2507 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f600) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3752 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee20d0, 0xc01512f580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3793 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce680, 0xc013cc0b80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852554 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 3812 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e5968f0, 0xc014f62780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 92510 [select]: | |
| net/http.(*persistConn).writeLoop(0xc017694fc0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2502 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc105fd7f38, {0x354b380, 0xc02122f410}, 0x1, 0xc026f4ee40) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0xb0, 0xc105fd7f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f400) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3799 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcfc70, 0xc013bffa00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2232 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b831a8, 0x130) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b89e0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83180, 0xc013a002e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013071b90) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020725a40}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013071bf8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013071b90, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713c20, 0xc003aa37b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 2234 [sync.Cond.Wait, 1635 minutes]: | |
| sync.runtime_notifyListWait(0xc012b832e8, 0xa1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc041d21cc0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b832c0, 0xc013a008a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a04630) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020725f20}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a04698, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a04630, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713cc0, 0xc04d216000) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 3794 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce750, 0xc013cc0980) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3307 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0137883d0, 0x303) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776ae0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).process(0xc013788400, {0x35a0c90, 0xc0135d1e00}, 0xc0136f1d10) | |
| /app/discovery/kubernetes/pod.go:107 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1() | |
| /app/discovery/kubernetes/pod.go:98 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run | |
| /app/discovery/kubernetes/pod.go:97 +0x1d2 | |
| goroutine 2525 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2278 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2630 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 9709645 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05b6eca90, 0xc0beaf55c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2115 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01392fa40, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f829200) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2461 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315ea0, 0xc01b7bc600) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2277 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f8310, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3809 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596820, 0xc014f62e80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2304 [sync.Cond.Wait, 2624 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1f68, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1f40, 0xc013a881a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a8a090) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6ebd0}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a8a0f8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a8a090, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e5a0, 0xc003a897b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 93292 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05e3ef440) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3677 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0197c1110, 0xc017164b80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 110439 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc022c791e0, 0xc0f9e23680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2198 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc00f4c1200, {0x35a0c90, 0xc0135d1ac0}, 0xc027b0bda0) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc00f4c1200}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2364 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d0a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc06e2470c0}, 0xc003b17ce8, 0xc013a3f020, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d0a0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c319a0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e05200, {0x354b360, 0xc012c31900}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d0a0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10606334 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a738c9c0, 0xc0a0aa9940) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2951 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc013a02d10, 0x2cc) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013a3e2a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc013a1e500, {0x35a0c90, 0xc0137892c0}, 0xc011d26da0) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3845 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0203592b0, 0xc016f53a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2390 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe1c0, 0xc003b02fb0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 69623 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05cf75200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3781 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf380, 0xc0190d3580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3811 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e5960d0, 0xc014f62980) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3580 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4680, 0xc00d717f80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2403 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315b20, 0xc003ac77a8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2643 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 10849915 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc0985901c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc020794ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0985901b0, {0xc083446800, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc083446800, 0x217c472, 0xc003ac44f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0973f7180) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0973f7180) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0973f7180, {0x2748340, 0xc098615938}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc09861f440, {0xc09862e000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc097e69c20, 0xc003ac4638, {0x357ac00, 0xc098597c40}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0986224a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc098597c00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 3768 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596410, 0xc01691aa80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2466 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232fc00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2385 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 3815 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596b60, 0xc014f62180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 67115 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228aba00, 0xc0a8a6cb80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10818888 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc00d4461c8, 0x7) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc00d4461b0, {0xc03c7d4c01, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc03c7d4c01, 0xc03c7d4c00, 0xc0171a1cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc03d6fcdc0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc03d6fcdc0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc03d6fcdc0, {0x2748340, 0xc08d6440f0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03e720630, {0xc0a90c3800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00d1433b0, 0xc0392e1a70, {0x357ac00, 0xc08d63ce80}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc015463940) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0c5003100) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 3689 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185efad0, 0xc003ac2f78) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2295 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1c48, 0x10a) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc04d3bd140) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1c20, 0xc0104bd310) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c66c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fde3f80}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c6728, 0x3b9aca00, 0x0, 0xc0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c66c0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e320, 0xc04d2165a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 68438 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b4b68, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0193bf180, 0xc022563000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0193bf180, {0xc022563000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0193bf180, {0xc022563000, 0xc032f53400, 0xc0a6064f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc050f7fd10, {0xc022563000, 0xc036d8c820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc077644480, {0xc022563000, 0xc030d0ac00, 0xc0a6064d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc000dcdd40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc000dcdd40, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc077644480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3880 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202792b0, 0xc01424e600) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2302 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1e28, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e36a0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1e00, 0xc0104bdbb0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c75f0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6e6f0}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c7658, 0x3b9aca00, 0x0, 0x40, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c75f0, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e500, 0xc0043d6f68) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 2197 [sync.Cond.Wait, 19 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1ba8, 0x1e2) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0570b1940) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1b80, 0xc0104bd010) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c61b0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fde3ce0}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c6218, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c61b0, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00ecb1900, 0xc00f829800) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 2748 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2463 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488ac40, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc05dba9d80}, 0xc013a3bce8, 0xc01602e420, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488ac40, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a08640) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7580, {0x354b360, 0xc013a085a0}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488ac40, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2404 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2405 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765ce0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc020b2d640}, 0xc105febce8, 0xc013e7f380, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765ce0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035cc0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6c80, {0x354b360, 0xc013035c20}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765ce0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2499 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f180) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2186 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 10832160 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06271c360) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2386 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d180, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02b040740}, 0xc003b15ce8, 0xc013d4a600, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d180, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31ae0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e05380, {0x354b360, 0xc012c31a40}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d180, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2567 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2465 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0043daf38, {0x354b380, 0xc020f6f0e0}, 0x1, 0xc0285aa660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc004694ea0, 0x3b9aca00, 0x0, 0x40, 0xc0043daf88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232fc00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10843386 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc05d576948, 0x2) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc05d576930, {0xc00b251801, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc00b251801, 0x6, 0xc01449fcf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05ceb3cc0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05ceb3cc0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05ceb3cc0, {0x2748340, 0xc021fb97e8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0f8bb5470, {0xc0e20e3800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05db5dc20, 0xc001fb5b90, {0x357ac00, 0xc021fc0e00}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05dbcf1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc05dba9d80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 3678 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0197c12b0, 0xc017164980) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3817 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783860, 0xc014f21d80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2296 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1ce8, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0305b6900) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1cc0, 0xc0104bd5f0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c6bd0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6e210}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c6c38, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c6bd0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e280, 0xc003ac47b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2298 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232f900, {0x35a0c90, 0xc0135d1e80}, 0xc028350060) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232f900}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2355 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe070, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10606319 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc052d825b0, 0xc06f3cc600) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2301 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 10852947 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3cc8, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04719ec00, 0xc07f544c90, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc04719ec00, {0x35a0d00, 0xc041d80360}, {0xc02074b2f8, 0x40ce34}, {0x354bea0, 0xc0441224c0}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc04719ec00, {0x35a0d00, 0xc041d80360}, {0x35c3098, 0x0}, {0x35c3098, 0xc07f544c60}, 0xc02074b4e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc041d80360}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc041d80360}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc07f544c60}, 0xc006ce5250, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc04719eb80, {0x35a0d00, 0xc041d80360}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc041d80360, {0x35a0d00, 0xc041d80360}, 0xc0b49a52a8, 0x100000000b0fcc6) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc04719eb80, {0x35a0d00, 0xc041d80360}, {0x357b3f8, 0xc07f544c60}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc04719eb80, {0x35a0d00, 0xc041d80360}, {0xc040d0cd00, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc01978eea0, {0x35a0d00, 0xc041d80360}, {0x2c655e4, 0xc040d0cce0}, {0xc003179b00, 0xc072d06c60}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc041d80360}, {0x2c655e4, 0x3}, {0xc003179b00, 0x13}, {0xc017c69590, 0x0}, 0xc01978aec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc041d80360}, {0x2c655e4, 0x3}, {0xc003179b00, 0x13}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc041d80360, {0x35a0d00, 0xc041d80360}, {0x2c655e4, 0x416af5}, {0xc003179b00, 0x20300f}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f53900, {0x35a0d00, 0xc041d80360}, {{}, 0x0, {0xc0499e3e80, 0x5}, {0xc003179b00, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0xc0e0cc9980, 0xc0ecafcfd0) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 2362 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe000, 0xc003a57758) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2397 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2495 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232ef00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2253 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83748, 0x1a7) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0305b6b60) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83720, 0xc013a01d20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a2ca20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122f080}, 0x1, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a2ca88, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a2ca20, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e140, 0xc105fd1fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:471 +0x2e0f | |
| goroutine 2195 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1a68, 0x15d) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b8e60) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1a40, 0xc0104bca50) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0b8d8d710) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fde3800}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0b8d8d778, 0x3b9aca00, 0x0, 0xc0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0b8d8d710, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00ecb1860, 0xc00f988300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 2196 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1b08, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0305b6680) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1ae0, 0xc0104bcd30) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0b8d8dc20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fde3a70}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0b8d8dc88, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0b8d8dc20, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00ecb17c0, 0xc00f829f00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2305 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232fb80, {0x35a0c90, 0xc0135d1f00}, 0xc0283500c0) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232fb80}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2375 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315ab0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2308 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776d20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 3779 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce5b0, 0xc0194f6440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2542 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2451 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488aa80, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc01dd97fc0}, 0xc013a11ce8, 0xc014a7e900, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488aa80, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a083c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7340, {0x354b360, 0xc013a08320}, 0x1, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488aa80, 0xc028350c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3879 [select]: | |
| net/http.(*Transport).getConn(0xc032f52280, 0xc025607c80, {{}, 0x0, {0xc03c825e00, 0x5}, {0xc02fdca108, 0x11}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f52280, 0xc068126500) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc083380d80) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02da161b0, 0xc068126400) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc068126400, {0x3548b80, 0xc02da161b0}, {0x2bdc000, 0xc01c8ff601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02da161e0, 0xc068126400, {0x0, 0xc09ed53758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02da161e0, 0xc068126400) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc0205ca140, {0x35a0d00, 0xc0255bc2a0}, {0x3544a40, 0xc083380cc0}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc0202791e0, {0xeda5786ec, 0x4d8e460, 0x4d8e460}, {0xeda5786ec, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202791e0, 0xc0139b7280) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10600322 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04a133860, 0xc043625080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2127 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc034182c40, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f16cf80) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2752 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc013788150, 0x233) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc0137768a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232f100, {0x35a0c90, 0xc0135d1c80}, 0xc013776c00) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 2504 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003ac0f38, {0x354b380, 0xc02122f500}, 0x1, 0xc026f4eea0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01aa54300, 0x3b9aca00, 0x0, 0x0, 0xc003ac0f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f480) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3775 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596680, 0xc014f63280) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2303 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1ec8, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0305b6700) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1ea0, 0xc0104bdeb0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c7b00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6e960}, 0x1, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c7b68, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c7b00, 0xc0283516e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e460, 0xc008582768) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2368 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0137fe0e0, 0xc003ac1758) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2376 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2356 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 67907 [select]: | |
| net/http.(*persistConn).writeLoop(0xc094141d40) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3783 [select]: | |
| net/http.(*Transport).getConn(0xc032f53900, 0xc03976ae80, {{}, 0x0, {0xc043b35380, 0x5}, {0xc020225c98, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f53900, 0xc039712700) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0f563fb00) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02d9c1c20, 0xc039712600) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc039712600, {0x3548b80, 0xc02d9c1c20}, {0x2bdc000, 0xc01e82b601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02d9c1c50, 0xc039712600, {0x0, 0xc09ed99758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02d9c1c50, 0xc039712600) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc0196b7f80, {0x35a0d00, 0xc03973b9e0}, {0x3544a40, 0xc0f563fa40}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01cfce9c0, {0xeda5786ee, 0x4d8e460, 0x4d8e460}, {0xeda5786ee, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce9c0, 0xc015845700) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3878 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279040, 0xc0139b7480) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2950 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00fd0f4d0, 0x350) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc012bac660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc00fd2b180, {0x35a0c90, 0xc0135d19c0}, 0x82b825) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3877 [select]: | |
| net/http.(*Transport).getConn(0xc032f52280, 0xc025606d00, {{}, 0x0, {0xc032cb0080, 0x5}, {0xc00de2ab28, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f52280, 0xc0fa4d6a00) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0b46ba960) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02da161b0, 0xc0fa4d6900) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc0fa4d6900, {0x3548b80, 0xc02da161b0}, {0x2bdc000, 0xc01b5c5601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02da161e0, 0xc0fa4d6900, {0x0, 0xc09edc3758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02da161e0, 0xc0fa4d6900) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01e425900, {0x35a0d00, 0xc0254f91a0}, {0x3544a40, 0xc0b46ba8a0}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc020278820, {0xeda5786e9, 0x4d8e460, 0x4d8e460}, {0xeda5786e9, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278820, 0xc0139b7700) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2224 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 1265 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6980, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc035978100, 0xc003c2e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc035978100, {0xc003c2e000, 0xb1f9, 0xb1f9}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc035978100, {0xc003c2e000, 0xc003c2e005, 0xc7}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0136c5068, {0xc003c2e000, 0x400, 0xc003b1b7f0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc039bd0a68, {0xc003c2e000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc003a6c5f8, {0x3544be0, 0xc039bd0a68}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc003a6c380, {0x354b880, 0xc0136c5068}, 0xb1f9) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003a6c380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc003a6c380, {0xc0118fe000, 0x1000, 0xc003b1bcb0}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc003b59920, {0xc0341649e0, 0x9, 0xad}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc003b59920}, {0xc0341649e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0341649e0, 0x9, 0xc001112690}, {0x35449e0, 0xc003b59920}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0341649a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003b1bf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc01371be00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3770 [select]: | |
| net/http.(*Transport).getConn(0xc032f59e00, 0xc02d7ace00, {{}, 0x0, {0xc043cae4c0, 0x5}, {0xc023544b28, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f59e00, 0xc02d850000) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0877eb9e0) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02dda0c00, 0xc05a0a9500) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc05a0a9500, {0x3548b80, 0xc02dda0c00}, {0x2bdc000, 0xc01e4b3601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02dda0c30, 0xc05a0a9500, {0x0, 0xc09edcb758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02dda0c30, 0xc05a0a9500) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01c781740, {0x35a0d00, 0xc02d73bb00}, {0x3544a40, 0xc0877eb920}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01c7836c0, {0xeda5786ed, 0x4d8e460, 0x4d8e460}, {0xeda5786ed, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c7836c0, 0xc01691a480) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3729 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f5110, 0xc017277800) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3792 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce410, 0xc013cc0f80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3692 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c782000, 0xc003a50f78) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2239 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83388, 0x12d) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b9800) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83360, 0xc013a00b90) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a04b40) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122e1b0}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a04ba8, 0x3b9aca00, 0x0, 0x40, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a04b40, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713e00, 0xc04d217b90) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 6205929 [select]: | |
| net/http.(*persistConn).writeLoop(0xc088c05200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3616 [select]: | |
| net/http.(*Transport).getConn(0xc032f53900, 0xc040cf7740, {{}, 0x0, {0xc0499e3e80, 0x5}, {0xc003179b00, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f53900, 0xc041d7f100) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc07f544bd0) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02d9c1c20, 0xc041d7f000) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc041d7f000, {0x3548b80, 0xc02d9c1c20}, {0x2bdc000, 0xc01ed97601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02d9c1c50, 0xc041d7f000, {0x0, 0xc09ee03758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02d9c1c50, 0xc041d7f000) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc0196b7980, {0x35a0d00, 0xc041d80360}, {0x3544a40, 0xc07f544b10}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01cfce270, {0xeda5786ef, 0x4d8e460, 0x4d8e460}, {0xeda5786ef, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce270, 0x0) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2649 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2120 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc013ff7c00, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f301e00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2107 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01398c9a0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x0) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2106 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc034182700, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x8) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2225 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b82fc8, 0x135) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc04013a080) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b82fa0, 0xc011d27a40) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013070c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc0207252f0}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013070cc8, 0x3b9aca00, 0x0, 0xc0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013070c60, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713a40, 0xc00f988500) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 10832159 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0604ea0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05cbff880, 0xc015bf4000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05cbff880, {0xc015bf4000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05cbff880, {0xc015bf4000, 0xc032f59b80, 0xc01e611f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0682ce3d0, {0xc015bf4000, 0xc02fe6a4e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06271c360, {0xc015bf4000, 0xc09cbfbf20, 0xc01e611d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04ae77500) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04ae77500, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06271c360) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2240 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83428, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc04d3bd0c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83400, 0xc013a00e80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a05050) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122e420}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a050b8, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a05050, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713d60, 0xc003b047b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2242 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232f100, {0x35a0c90, 0xc0135d1c80}, 0xc027b0bec0) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232f100}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 3860 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020359110, 0xc016dfe700) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 92100 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b50d8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0781c0c00, 0xc0ffcdf400, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0781c0c00, {0xc0ffcdf400, 0xbcf, 0xbcf}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0781c0c00, {0xc0ffcdf400, 0xbca, 0xc07f5ac140}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc054450fa0, {0xc0ffcdf400, 0xc0ffcdf400, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc036ce8828, {0xc0ffcdf400, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc07f59a5f8, {0x3544be0, 0xc036ce8828}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc07f59a380, {0x7f09e11af0a0, 0xc07f5a8600}, 0xbcf) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07f59a380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc07f59a380, {0xc07c834000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc07f5bd4a0, {0xc042b2db60, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc07f5bd4a0}, {0xc042b2db60, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc042b2db60, 0x9, 0xc001b728d0}, {0x35449e0, 0xc07f5bd4a0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc042b2db20) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01ac74f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc07f5c2000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 2313 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02eccbe00) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02eccbe00, {0x35a0c90, 0xc0137892c0}, 0xc028765800) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2112 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc034182460, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f829d00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2113 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01446a9a0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f829a00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2108 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138cea80, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x0) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2429 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488a9a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc040ac2bc0}, 0xc013a17ce8, 0xc013d4a300, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488a9a0, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a08230) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7180, {0x354b360, 0xc013a08190}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488a9a0, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2309 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc013a8c028, 0x2d3e) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0986225c0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc013a8c000, 0xc013a88490) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a8a5a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6ee40}, 0x1, 0xc028351980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a8a608, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a8a5a0, 0xc028351980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e640, 0xc003b05fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:471 +0x2e0f | |
| goroutine 2310 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc013788680, {0x35a0c90, 0xc0135d1f80}, 0xc028350180) | |
| /app/discovery/kubernetes/pod.go:103 +0x1ef | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548ea0, 0xc013788680}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2279 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0147657a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02fb47d40}, 0xc000853ce8, 0xc01707d440, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0147657a0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0130354a0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d64c0, {0x354b360, 0xc013035400}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0147657a0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 132194 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0763ca270, 0xc0f43ae640) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 146382 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bafb0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0236b0000, 0xc00c0c6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0236b0000, {0xc00c0c6000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0236b0000, {0xc00c0c6000, 0x3c6f, 0xc0d6fea400}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc067ac9410, {0xc00c0c6000, 0xc00c0c6000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0421473c8, {0xc00c0c6000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc1039bdaf8, {0x3544be0, 0xc0421473c8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc1039bd880, {0x7f09e11af0a0, 0xc03b7fd700}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc1039bd880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc1039bd880, {0xc012e18000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc079e8fbc0, {0xc00fd3e200, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc079e8fbc0}, {0xc00fd3e200, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00fd3e200, 0x9, 0xc001545500}, {0x35449e0, 0xc079e8fbc0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00fd3e1c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0104f4f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc014f44c00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3777 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce340, 0xc0327ef200) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 146406 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733b28, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0f9e9de80, 0xc09333c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0f9e9de80, {0xc09333c000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0f9e9de80, {0xc09333c000, 0x3c6f, 0xc088d10660}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0677e1bb8, {0xc09333c000, 0xc09333c000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc039aefad0, {0xc09333c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc113acfe78, {0x3544be0, 0xc039aefad0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc113acfc00, {0x7f09e11af0a0, 0xc02171bf40}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc113acfc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc113acfc00, {0xc10c590000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc056286120, {0xc0115da660, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc056286120}, {0xc0115da660, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0115da660, 0x9, 0xc00195c150}, {0x35449e0, 0xc056286120}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0115da620) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e617f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc01e6cf200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10806261 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05a6cc750, 0xc058c6b700) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68592 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08c9a06c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10607238 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f480, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0043ee880, 0xc0345fcd00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0043ee880, {0xc0345fcd00, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0043ee880, {0xc0345fcd00, 0x812, 0xc05f0b83c0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc022ebf680, {0xc0345fcd00, 0xc0345fcd00, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc042d056c8, {0xc0345fcd00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0b0b39e78, {0x3544be0, 0xc042d056c8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0b0b39c00, {0x7f09e11af0a0, 0xc0b1b70c40}, 0x817) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0b0b39c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0b0b39c00, {0xc0b1f18000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0e22e7c80, {0xc047add8c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0e22e7c80}, {0xc047add8c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc047add8c0, 0x9, 0xc0010c7830}, {0x35449e0, 0xc0e22e7c80}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc047add880) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01c7a6f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc00c39a600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3956722 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc07b5bbc70, 0xc040cd38c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2233 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83248, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0305b6820) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b83220, 0xc013a005c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a04120) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020725cb0}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a04188, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a04120, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713b80, 0xc003b06fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 2294 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776ba0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2297 [sync.Cond.Wait, 2603 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb1d88, 0xa) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0037f7fe0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb1d60, 0xc0104bd8d0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0138c70e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020f6e480}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0138c7148, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0138c70e0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e3c0, 0xc003b07fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 3786 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf1e0, 0xc015845000) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2238 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0137768a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2241 [sync.Cond.Wait, 9 minutes]: | |
| sync.runtime_notifyListWait(0xc012b834c8, 0x77) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc03b2248a0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b834a0, 0xc013a011a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a05560) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122e6c0}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a055c8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a05560, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713ea0, 0xc003b26fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 3774 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e5964e0, 0xc014f63480) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3755 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee25b0, 0xc01512f580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2471 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 113976 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0411a0ea0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2121 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01401cb60, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f301c00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2111 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01401c1c0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x0) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2157 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1ac0}, 0xc012fb4640, 0xc027b0bda0) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 10852722 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06056c8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc01cb65800, 0xc0cda56000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc01cb65800, {0xc0cda56000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc01cb65800, {0xc0cda56000, 0xc032f59180, 0xc01d8c8f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013909418, {0xc0cda56000, 0xc02b5ff6c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc062ee4ea0, {0xc0cda56000, 0xc05fae2360, 0xc01d8c8d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0255bc4e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0255bc4e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc062ee4ea0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3814 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783ba0, 0xc014f62380) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2124 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138ced20, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf5800) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3680 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0197c15f0, 0xc017164780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2123 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc106001b20, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf5c00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3592 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185eedd0, 0xc017276680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2105 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc034182620, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0x0) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2211 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc01023c850, 0xc0135d1940) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2179 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b82de8, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc04d3bcca0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b82dc0, 0xc011d270c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d1c0bcb0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020724570}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0d1c0bd18, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0d1c0bcb0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b82b40, 0xc01a1c6a20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 3875 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279790, 0xc0139b7d80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2353 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc010243300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2245 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013776960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2291 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc013788400, {0x35a0c90, 0xc0135d1e00}, 0xc028350000) | |
| /app/discovery/kubernetes/pod.go:103 +0x1ef | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548ea0, 0xc013788400}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 3841 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020358c30, 0xc016f96900) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3598 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ef380, 0xc003a7cd80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 1548 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc010423388, 0x128) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e3980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc010423360, 0xc013246520) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0104415f0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec284d48) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fbbfb90}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc010441658, 0x3b9aca00, 0x0, 0x40, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0104415f0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc010423220, 0xc027b0bc20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 3594 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185ef110, 0xc017276680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2177 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc012bac660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2530 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec190b10) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003b20f38, {0x354b380, 0xc020aace70}, 0x1, 0xc026179740) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x122caf30, 0x3b9aca00, 0x0, 0xb0, 0xc003b20f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc013a1e600) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3967427 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb608, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc03386a500, 0xc0434f2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc03386a500, {0xc0434f2000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc03386a500, {0xc0434f2000, 0x3ccb, 0xc0422c37e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc07d100168, {0xc0434f2000, 0xc0434f2000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01ee0da58, {0xc0434f2000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc042da0978, {0x3544be0, 0xc01ee0da58}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc042da0700, {0x7f09e11af0a0, 0xc03dd94e00}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc042da0700, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc042da0700, {0xc027217000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc03915d260, {0xc032569b60, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc03915d260}, {0xc032569b60, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc032569b60, 0x9, 0xc001191a10}, {0x35449e0, 0xc03915d260}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc032569b20) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0037cef98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0360e7500) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 2510 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0104239c8, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b88e0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0104239a0, 0xc013247400) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013900a20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec190b10) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020aac8d0}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013900a88, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013900a20, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b837c0, 0xc028765800) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 68591 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644a50, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a1218380, 0xc0cd18a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a1218380, {0xc0cd18a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a1218380, {0xc0cd18a000, 0xc032f58dc0, 0xc01988ff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc001c990d8, {0xc0cd18a000, 0xc054f03d40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08c9a06c0, {0xc0cd18a000, 0xc094933920, 0xc01988fd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a6616a20) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a6616a20, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08c9a06c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2498 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003b20738, {0x354b380, 0xc02122f3b0}, 0x1, 0xc026f4ed80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00279d730, 0x3b9aca00, 0x0, 0x97, 0xc003b20788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f180) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2290 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b82f28, 0x2e8) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00e9b2360) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b82f00, 0xc011d27760) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013070750) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020724b40}, 0x1, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0130707b8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013070750, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc01379e1e0, 0xc003ba3f78) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:471 +0x2e0f | |
| goroutine 2213 [select, 3 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01447a0e0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0113ebcc0}, 0xc003bb9ce8, 0xc013a3faa0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01447a0e0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013248a50) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc017621e00, {0x354b360, 0xc0132489b0}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01447a0e0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2212 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2228 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc01232ec00, {0x35a0c90, 0xc0135d1b40}, 0xc027b0be00) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc01232ec00}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 1551 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc010243280, {0x35a0c90, 0xc0135d1940}, 0xc027b0bc20) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc010243280}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2180 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc012b82e88, 0x2c39) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc097cc4800) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b82e60, 0xc011d273a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013070240) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc0207247e0}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0130702a8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013070240, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b82c80, 0xc003ac67b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 3306 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0137882d0, 0x246) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776a20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).process(0xc013788300, {0x35a0c90, 0xc0135d1d80}, 0xc013776900) | |
| /app/discovery/kubernetes/pod.go:107 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1() | |
| /app/discovery/kubernetes/pod.go:98 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run | |
| /app/discovery/kubernetes/pod.go:97 +0x1d2 | |
| goroutine 10852811 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b5478, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc025094f80, 0xc08c112000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc025094f80, {0x35a0d00, 0xc02d85e000}, {0xc013a392f8, 0x40ce34}, {0x354bea0, 0xc07f6b2aa0}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc025094f80, {0x35a0d00, 0xc02d85e000}, {0x35c3098, 0x0}, {0x35c3098, 0xc0877ebfb0}, 0xc013a394e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0877ebfb0}, 0xc0d1d7c040, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc025094f00, {0x35a0d00, 0xc02d85e000}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc02d85e000, {0x35a0d00, 0xc02d85e000}, 0xc0b49a52a8, 0x100000000b0fcc6) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc025094f00, {0x35a0d00, 0xc02d85e000}, {0x357b3f8, 0xc0877ebfb0}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc025094f00, {0x35a0d00, 0xc02d85e000}, {0xc02d1f5840, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc01978eea0, {0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0xc02d1f5820}, {0xc023544be8, 0xc0239808f0}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0x3}, {0xc023544be8, 0x12}, {0xc017c69590, 0x0}, 0xc01978aec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0x3}, {0xc023544be8, 0x12}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc02d85e000, {0x35a0d00, 0xc02d85e000}, {0x2c655e4, 0x0}, {0xc023544be8, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f53900, {0x35a0d00, 0xc02d85e000}, {{}, 0x0, {0xc045879f40, 0x5}, {0xc023544be8, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x35a16a0, 0xc0ecafcb00) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 10819706 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc00e614f48, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc050199ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc00e614f30, {0xc0d5402001, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc0d5402001, 0xc06c1de000, 0xc050199cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc067ea2280) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc067ea2280) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc067ea2280, {0x2748340, 0xc0a36af4a0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06f426ab0, {0xc02e2df400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0139f0410, 0xc0a2f72000, {0x357ac00, 0xc0afaca500}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05a6ce2c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc073191100) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 92852 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730850, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b3bbed80, 0xc0b5050000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b3bbed80, {0xc0b5050000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b3bbed80, {0xc0b5050000, 0xc032f52f00, 0xc020458f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0334d4db0, {0xc0b5050000, 0xc07cb46680, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc039c1e5a0, {0xc0b5050000, 0xc068878a20, 0xc020458d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0b3919f80) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0b3919f80, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc039c1e5a0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2517 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2516 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f8b60, 0x11e3dbbe) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2651 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 10851827 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc01df10048, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01a259ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc01df10030, {0xc05b22d800, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x1}, {0xc05b22d800, 0x416f26, 0xc003aa74f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc09dd8b680) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc09dd8b680) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc09dd8b680, {0x2748340, 0xc01dee5a70}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0602e02a0, {0xc00c7ae000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01df140a0, 0xc003aa7638, {0x357ac00, 0xc01df16000}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01defc800) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01dd97fc0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2335 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01447a460, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc06f2aec40}, 0xc003b35ce8, 0xc0150e8900, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01447a460, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0132493b0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc01aea8600, {0x354b360, 0xc013249310}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01447a460, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2508 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc013a3e2a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2063 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc00f4cd600, {0x35a0c90, 0xc0135d1a40}, 0xc027b0bd40) | |
| /app/discovery/kubernetes/pod.go:103 +0x1ef | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548ea0, 0xc00f4cd600}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 2509 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b839c8, 0x172) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc04d3bce60) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b839a0, 0xc013a44270) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013a2cf30) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc02122fc50}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013a2cf98, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013a2cf30, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b83860, 0xc003b24f68) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 2114 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0341827e0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f829400) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2227 [sync.Cond.Wait, 2603 minutes]: | |
| sync.runtime_notifyListWait(0xc012b83108, 0x4c) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc102335e40) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b830e0, 0xc013a00000) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013071680) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc0207257d0}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0130716e8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013071680, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc013713ae0, 0xc0011a8fb8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 2231 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0137767e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 3874 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202795f0, 0xc013bfe180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3581 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f48f0, 0xc01882f200) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2561 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2518 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488ae00, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0be1d2e80}, 0xc013a37ce8, 0xc014a7f260, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488ae00, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a08be0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d77c0, {0x354b360, 0xc013a08b40}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488ae00, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3797 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcfa00, 0xc013bffc00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2511 [sync.Cond.Wait, 58 minutes]: | |
| sync.runtime_notifyListWait(0xc010423a68, 0xc7) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc06d748360) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc010423a40, 0xc0132476f0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013900f30) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec190b10) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020aacb40}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc013900f98, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013900f30, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b83900, 0xc105fd77b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 2512 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc013a1e500, {0x35a0c90, 0xc0137892c0}, 0xc028765800) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc013a1e500}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 3795 [select]: | |
| net/http.(*Transport).getConn(0xc032f53900, 0xc0256ddf80, {{}, 0x0, {0xc035f47b80, 0x5}, {0xc0201403f0, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f53900, 0xc0a968e300) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc0d7689620) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02d9c1c20, 0xc0a968e200) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc0a968e200, {0x3548b80, 0xc02d9c1c20}, {0x2bdc000, 0xc01c3ab601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02d9c1c50, 0xc0a968e200, {0x0, 0xc09f077758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02d9c1c50, 0xc0a968e200) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01e424340, {0x35a0d00, 0xc020bdfb60}, {0x3544a40, 0xc0d7689500}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01cfceea0, {0xeda5786eb, 0x4d8e460, 0x4d8e460}, {0xeda5786eb, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfceea0, 0xc013cc0780) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2333 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f8bd0, 0xc0043d87b0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10823929 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc033858c00, 0xc068971800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce8c6, 0xc0a6425560) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 1550 [sync.Cond.Wait, 242 minutes]: | |
| sync.runtime_notifyListWait(0xc0104234c8, 0x2e) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01f21d1e0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0104234a0, 0xc013246b20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc013900090) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec190b10) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020aac090}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0139000f8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc013900090, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0104232c0, 0xc01a1c6ba0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:449 +0x1e2f | |
| goroutine 1549 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc010423428, 0x107) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e33e0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc010423400, 0xc013246840) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc010441b00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec284d48) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fbbfe00}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc010441b68, 0x3b9aca00, 0x0, 0x80, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc010441b00, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc010423180, 0xc003a1f7b8) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:448 +0x1d6f | |
| goroutine 3304 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0137884d0, 0x1ec) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776ba0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232f900, {0x35a0c90, 0xc0135d1e80}, 0xc0132464e0) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3959098 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732700, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0443cf300, 0xc09eb7a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0443cf300, {0xc09eb7a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0443cf300, {0xc09eb7a000, 0xc032f58780, 0xc0037d1f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0ee2e6138, {0xc09eb7a000, 0xc02b05bba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc040d7d200, {0xc09eb7a000, 0xc06026db00, 0xc0037d1d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc084367e60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc084367e60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc040d7d200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3984 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a5db8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02148c780, 0xc0214cf000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02148c780, {0xc0214cf000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02148c780, {0xc0214cf000, 0xc032c9bb80, 0xc01c7a8f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01aac51c0, {0xc0214cf000, 0xc02c49b520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc057821c20, {0xc0214cf000, 0xc020e598c0, 0xc01c7a8d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc021351320) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc021351320, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc057821c20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2523 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d8a8) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00398df38, {0x354b380, 0xc02131c960}, 0x1, 0xc026f4faa0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01b7bcd80, 0x3b9aca00, 0x0, 0xe8, 0xc00398df88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc013a1e580) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2524 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc013a1e580) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3844 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0203588f0, 0xc016f96300) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3305 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc013788090, 0x2bf) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc0137767e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232ee80, {0x35a0c90, 0xc0135d1c00}, 0xc012c85ce0) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 10851909 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730bf0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05b3b0180, 0xc0a318a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05b3b0180, {0xc0a318a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05b3b0180, {0xc0a318a000, 0xc032f53cc0, 0xc02080ff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc09748e030, {0xc0a318a000, 0xc096edfd40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc027370000, {0xc0a318a000, 0xc063570300, 0xc02080fd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc01e0ae960) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc01e0ae960, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc027370000) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2185 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f82a0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2287 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003b24738, {0x354b380, 0xc020724a80}, 0x1, 0xc02790a6c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc07057f0e0, 0x3b9aca00, 0x0, 0xa0, 0xc003b24788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc00f4c1280) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2187 [select, 6 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0147655e0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc02af97f80}, 0xc003b33ce8, 0xc01324bda0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0147655e0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035220) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6280, {0x354b360, 0xc013035180}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0147655e0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2056 [select]: | |
| github.com/prometheus/prometheus/scrape.(*Manager).reloader(0xc0006ccf60) | |
| /app/scrape/manager.go:174 +0x145 | |
| created by github.com/prometheus/prometheus/scrape.(*Manager).Run | |
| /app/scrape/manager.go:148 +0x65 | |
| goroutine 2181 [chan receive, 2624 minutes]: | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc00fd2b180, {0x35a0c90, 0xc0135d19c0}, 0xc027b0bc80) | |
| /app/discovery/kubernetes/endpoints.go:150 +0x2e5 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func24({0x3548e40, 0xc00fd2b180}) | |
| /app/discovery/kubernetes/kubernetes.go:580 +0x75 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:578 +0x8cf | |
| goroutine 3684 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c782750, 0xc0177d3dc0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3753 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee21a0, 0xc01512f280) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2282 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003a54738, {0x354b380, 0xc020724a20}, 0x1, 0xc02790a600) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01adc1300, 0x3b9aca00, 0x0, 0xa0, 0xc003a54788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc00fd2b200) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2117 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc106001960, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f828900) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2178 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc012b82d48, 0x189) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0239e3d40) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc012b82d20, 0xc011d26de0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d1c0b7a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc020724300}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0d1c0b808, 0x3b9aca00, 0x0, 0xc0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0d1c0b7a0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc012b82be0, 0xc027b0bc80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:447 +0x1caf | |
| goroutine 3772 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e5961a0, 0xc016dfe100) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5262878 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d16508, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b3786200, 0xc03e50d000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b3786200, {0xc03e50d000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b3786200, {0xc03e50d000, 0xc032f532c0, 0xc003a38f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc002265d78, {0xc03e50d000, 0xc053e6e820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03a023e60, {0xc03e50d000, 0xc0b04a5380, 0xc003a38d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc08c08f140) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc08c08f140, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03a023e60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3970 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5fca8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02148c380, 0xc0214bf000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02148c380, {0xc0214bf000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02148c380, {0xc0214bf000, 0xc033083180, 0xc0183f1f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01aac5168, {0xc0214bf000, 0xc01b382340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0578218c0, {0xc0214bf000, 0xc020e58f00, 0xc0183f1d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc021350ae0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc021350ae0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0578218c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4047281 [select]: | |
| net/http.(*persistConn).writeLoop(0xc080f7ea20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2194 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc012b38420) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2329 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01447a380, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc072ec1d00}, 0xc003ad5ce8, 0xc013f16780, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01447a380, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013249270) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc01aea8500, {0x354b360, 0xc0132491d0}, 0x1, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01447a380, 0xc026f4f0e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3853 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020358000, 0xc016f52380) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3791 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce0d0, 0xc015844480) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2328 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 3813 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783ad0, 0xc014f62580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 66341 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003efc340, 0xc0bcf26380) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2223 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc01023c8c0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2116 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138ce8c0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f828f00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3789 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf6c0, 0xc0158d0d00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 67111 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab5f0, 0xc07b293180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 96218 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0908ce4e0, 0xc0a289c700) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 91276 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a2096d00, 0xc0180caf00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3805 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278340, 0xc013bff080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3762 [select]: | |
| net/http.(*Transport).getConn(0xc032f59e00, 0xc0257a0a40, {{}, 0x0, {0xc03692a000, 0x5}, {0xc023544438, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f59e00, 0xc0a5de3900) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc03c268cc0) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02dda0c00, 0xc0a5de3800) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc0a5de3800, {0x3548b80, 0xc02dda0c00}, {0x2bdc000, 0xc01c169601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02dda0c30, 0xc0a5de3800, {0x0, 0xc09f3ad758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02dda0c30, 0xc0a5de3800) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01e598100, {0x35a0d00, 0xc02552ec60}, {0x3544a40, 0xc03c268b40}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc01e596270, {0xeda5786eb, 0x4d8e460, 0x4d8e460}, {0xeda5786eb, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01e596270, 0xc0169bca80) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 9709559 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05a933ba0, 0xc0c3e47e60) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3863 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020359860, 0xc017415280) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3754 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01fee2270, 0xc01512f580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 145968 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a5cfc340, 0xc016f50fd0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3614 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcec30, 0xc003a1c778) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3778 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfce4e0, 0x0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3606 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc017db1ad0, 0xc017277a80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4004 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a5b00, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc021a6b400, 0xc099260000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc021a6b400, {0xc099260000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc021a6b400, {0xc099260000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01aa2c818, {0xc099260000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc02ff33c80, {0xc099260000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc00047ee40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc00047ee40, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc021f3c000, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 3317 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc013788590, 0x1cc) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776c60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232fb80, {0x35a0c90, 0xc0135d1f00}, 0x82b825) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3316 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0135d1fd0, 0x30d) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776720) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232ec00, {0x35a0c90, 0xc0135d1b40}, 0x82b825) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 3613 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcea90, 0xc005244780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 91536 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a9280b60, 0xc063df25c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3315 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc013788650, 0x2d9a) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776d20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).process(0xc013788680, {0x35a0c90, 0xc0135d1f80}, 0x82b825) | |
| /app/discovery/kubernetes/pod.go:107 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1() | |
| /app/discovery/kubernetes/pod.go:98 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run | |
| /app/discovery/kubernetes/pod.go:97 +0x1d2 | |
| goroutine 2325 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc010243380) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2324 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec190b10) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003d8ef38, {0x354b380, 0xc020aac2d0}, 0x1, 0xc0261790e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003bd6c4d, 0x3b9aca00, 0x0, 0xa0, 0x405d79) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc010243380) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 145868 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc021102f70, 0xc0f16951c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2653 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2321 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01447a2a0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc04eb3e140}, 0xc00116dce8, 0xc014a7f800, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01447a2a0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013248cd0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc01aea80c0, {0x354b360, 0xc013248c30}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01447a2a0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2377 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765b20, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc08d6df300}, 0xc101933ce8, 0xc014eca420, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765b20, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0130359f0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6a40, {0x354b360, 0xc013035950}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765b20, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2737 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2494 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003dbc738, {0x354b380, 0xc02122f350}, 0x1, 0xc026f4ec00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0141b9a10, 0x3b9aca00, 0x0, 0xa0, 0xc003dbc788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232ef00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2381 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315b90, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2382 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2383 [select, 6 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765c00, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0216e8580}, 0xc105fe9ce8, 0xc013a3ede0, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765c00, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035b80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6b40, {0x354b360, 0xc013035ae0}, 0x1, 0xc028350660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765c00, 0xc028350660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2521 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2327 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f8af0, 0xc006fd37b0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2192 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2357 [select, 5 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401cfc0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0824add80}, 0xc003b5bce8, 0xc013f17ce0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401cfc0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31860) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e050c0, {0x354b360, 0xc012c317c0}, 0x1, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401cfc0, 0xc0283513e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2283 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00fd2b200) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2286 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00fd2b280) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2288 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00f4c1280) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2338 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00f4c1080) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2342 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315f80, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2343 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2344 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765880, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc05b000e00}, 0xc10192fce8, 0xc0159fe180, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765880, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0130355e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d66c0, {0x354b360, 0xc013035540}, 0x1, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765880, 0xc0283510e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2544 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2540 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2130 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01398c700, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00eec3d00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3591 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0185eed00, 0xc0166c3580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2143 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc013ff7880, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00d717780) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2285 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003db6f38, {0x354b380, 0xc020724a50}, 0x1, 0xc02790a660) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc004d16850, 0x3b9aca00, 0x0, 0xa0, 0xc003db6f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc00fd2b280) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2191 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00f9f8230, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2496 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003db7f38, {0x354b380, 0xc02122f380}, 0x1, 0xc026f4ecc0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0141b98d0, 0x3b9aca00, 0x0, 0xa0, 0xc003db7f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232ef80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 92418 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605c38, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a1181080, 0xc0a28c4000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a1181080, {0xc0a28c4000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a1181080, {0xc0a28c4000, 0xc032c9bcc0, 0xc0a6067f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc05513f370, {0xc0a28c4000, 0xc0280c5d40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc045c62ea0, {0xc0a28c4000, 0xc0ef1a79e0, 0xc0a6067d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a287b6e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a287b6e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc045c62ea0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2536 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2219 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01447a1c0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc090289f40}, 0xc105fe7ce8, 0xc01324b680, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01447a1c0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013248b90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc017621fc0, {0x354b360, 0xc013248af0}, 0x1, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01447a1c0, 0xc024c079e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2218 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2217 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc01023c7e0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2445 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebeb9800) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc003df5f38, {0x354b380, 0xc020f6f080}, 0x1, 0xc0285aa5a0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc015c55d70, 0x3b9aca00, 0x0, 0xa0, 0xc003df5f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 95568 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04861dc70, 0xc0e49c4940) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2573 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2269 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401cee0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc041498f80}, 0xc003b5dce8, 0xc015250480, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401cee0, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c316d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e04fc0, {0x354b360, 0xc012c31630}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401cee0, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10849323 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc08eb12dc8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc08eb12db8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08eb12db0, {0xc0b5948c00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc009ba9410}, {0xc0b5948c00, 0x217c472, 0xc0085814f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc08e3352c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc08e3352c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc08e3352c0, {0x2748340, 0xc08ef00510}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08e332f60, {0xc08d9e9800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc08e330a50, 0xc008581638, {0x357ac00, 0xc08e0dd600}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc08e1007a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc08e0dd5c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2128 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc057d2c000, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f16c680) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2655 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2129 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01392f420, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00f16c480) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2062 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00ecb19c8, 0x110) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0368b9080) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00ecb19a0, 0xc0104bc770) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/delta_fifo.go:525 +0x198 | |
| k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0b8d8d170) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:183 +0x36 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec121e90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x16f8548, {0x354b380, 0xc01fde3590}, 0x1, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0b8d8d1d8, 0x3b9aca00, 0x0, 0x0, 0x7f09e61bf260) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*controller).Run(0xc0b8d8d170, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:154 +0x2fb | |
| k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00ecb1720, 0xc00ff88e80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:410 +0x449 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run | |
| /app/discovery/kubernetes/kubernetes.go:471 +0x2e0f | |
| goroutine 2273 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0147656c0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc073191100}, 0xc00116bce8, 0xc014a7fec0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0147656c0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d63c0, {0x354b360, 0xc0130352c0}, 0x1, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0147656c0, 0xc024c07980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2061 [chan receive]: | |
| k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc012b38360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:204 +0xa7 | |
| created by k8s.io/client-go/util/workqueue.newQueue | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:62 +0x1af | |
| goroutine 2628 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2172 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02eccab40) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02eccab40, {0x35a0c90, 0xc0135d1f00}, 0xc0283500c0) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2173 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1f00}, 0xc0136ea140, 0xc0283500c0) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2174 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02eccac80) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02eccac80, {0x35a0c90, 0xc0135d1f80}, 0xc028350180) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2538 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 67794 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd05b0, 0xc0e191c5a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2500 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc006bbf738, {0x354b380, 0xc02122f3e0}, 0x1, 0xc026f4ede0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0508442e0, 0x3b9aca00, 0x0, 0xa0, 0xc006bbf788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f200) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2421 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315c70, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2422 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2144 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc106001340, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ba0d700) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2136 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138cefc0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00e967600) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2139 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc057d2c0e0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf4600) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2626 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2137 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01401c9a0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00e967000) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2340 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc00f4c1300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2207 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00cfc9260, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2138 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01446b340, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00ecf5300) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2208 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2257 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401cd20, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc098597c00}, 0xc003b5fce8, 0xc013ab52c0, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401cd20, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31450) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e04d80, {0x354b360, 0xc012c313b0}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401cd20, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2475 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2339 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc006c4c738, {0x354b380, 0xc020724ae0}, 0x1, 0xc02790a780) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0706a86c8, 0x3b9aca00, 0x0, 0xa0, 0xc006c4c788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc00f4c1300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2261 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00cfc91f0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2262 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2263 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401ce00, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc08e0dd5c0}, 0xc003ad7ce8, 0xc013ab5e00, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401ce00, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c31590) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e04ec0, {0x354b360, 0xc012c314f0}, 0x1, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401ce00, 0xc02749a1e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2477 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2488 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc006c4f738, {0x354b380, 0xc02122f2c0}, 0x1, 0xc026f4eae0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc015c55ef0, 0x3b9aca00, 0x0, 0xa0, 0xc006c4f788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f780) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2267 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00cfc92d0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2268 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2442 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401d500, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc05d5a0940}, 0xc003b39ce8, 0xc0177ca300, 0xc028351980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401d500, 0xc028351980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a8e0a0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e058c0, {0x354b360, 0xc013a8e000}, 0x1, 0xc028351980) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401d500, 0xc028351980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2441 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 3893 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202789c0, 0xc005648e00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3892 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279450, 0xc005649100) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3891 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202796c0, 0xc005649300) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3890 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278dd0, 0xc005649500) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3889 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278c30, 0xc005649700) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3888 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278b60, 0xc005649a80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3887 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279a00, 0xc005649c80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3886 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279860, 0xc005649e80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3885 [select]: | |
| net/http.(*Transport).getConn(0xc032f52280, 0xc026d33480, {{}, 0x0, {0xc041ea24c0, 0x5}, {0xc0235446d8, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1372 +0x5d2 | |
| net/http.(*Transport).roundTrip(0xc032f52280, 0xc06e112300) | |
| /usr/local/go/src/net/http/transport.go:581 +0x774 | |
| net/http.(*Transport).RoundTrip(0x2974720, 0xc05cd74810) | |
| /usr/local/go/src/net/http/roundtrip.go:18 +0x19 | |
| github.com/prometheus/common/config.(*authorizationCredentialsFileRoundTripper).RoundTrip(0xc02da161b0, 0xc06e112200) | |
| /go/pkg/mod/github.com/prometheus/common@v0.32.1/config/http_config.go:511 +0x2d7 | |
| net/http.send(0xc06e112200, {0x3548b80, 0xc02da161b0}, {0x2bdc000, 0xc01ce8d601, 0x0}) | |
| /usr/local/go/src/net/http/client.go:252 +0x5d8 | |
| net/http.(*Client).send(0xc02da161e0, 0xc06e112200, {0x0, 0xc09f989758, 0x0}) | |
| /usr/local/go/src/net/http/client.go:176 +0x9b | |
| net/http.(*Client).do(0xc02da161e0, 0xc06e112200) | |
| /usr/local/go/src/net/http/client.go:725 +0x908 | |
| net/http.(*Client).Do(...) | |
| /usr/local/go/src/net/http/client.go:593 | |
| github.com/prometheus/prometheus/scrape.(*targetScraper).scrape(0xc01e425f00, {0x35a0d00, 0xc02745bc20}, {0x3544a40, 0xc05cd746f0}) | |
| /app/scrape/scrape.go:769 +0x6ff | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).scrapeAndReport(0xc020278f70, {0xeda5786ed, 0x4d8e460, 0x4d8e460}, {0xeda5786ed, 0x4d8e460, 0x4d8e460}, 0x0) | |
| /app/scrape/scrape.go:1279 +0x8a3 | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278f70, 0xc005786080) | |
| /app/scrape/scrape.go:1203 +0x351 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3884 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278ea0, 0xc005786400) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3883 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278a90, 0xc005786600) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3882 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279930, 0xc005786800) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3881 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279ba0, 0xc005786a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 439797 [select]: | |
| net/http.(*persistConn).writeLoop(0xc007f1ad80) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3615 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf110, 0xc0194f7a40) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3329 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc013788210, 0x2de) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc013776960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc01232f380, {0x35a0c90, 0xc0135d1d00}, 0xc013a44230) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 2505 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f480) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2501 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f200) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2497 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232ef80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2493 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232ed00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2491 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232ec80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2489 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f780) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 10850647 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0400ac348, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0400ac338) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0400ac330, {0xc0b78dda00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc0b78dda00, 0xc0045f7d60, 0xc0045f7cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05f130c80) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05f130c80) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05f130c80, {0x2748340, 0xc08cf467b0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0b5c8a7e0, {0xc096c80800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc060dd9630, 0xc0135d1d80, {0x357ac00, 0xc025e14340}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0223f8d80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc025e14300) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 2645 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2485 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01488ad20, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc025e14300}, 0xc013a3dce8, 0xc013e7ff20, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01488ad20, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013a087d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d7680, {0x354b360, 0xc013a08730}, 0x1, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01488ad20, 0xc028350ea0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2484 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2483 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315f10, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2506 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ebe7d2d0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc006fd5f38, {0x354b380, 0xc02122f710}, 0x1, 0xc026f4ef00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0140afd30, 0x3b9aca00, 0x0, 0xa0, 0xc006fd5f88) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc01232f600) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 68738 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08e4fa7e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 2640 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2417 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765ea0, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc0a6bb5280}, 0xc003bb7ce8, 0xc013f17560, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765ea0, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035f90) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6ec0, {0x354b360, 0xc013035ef0}, 0x1, 0xc028350960) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765ea0, 0xc028350960) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2416 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2415 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc012315ce0, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 3728 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4dd0, 0xc005dae380) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3727 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4c30, 0xc005dae680) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6205928 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733c10, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc03a69d180, 0xc0a1a17000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc03a69d180, {0xc0a1a17000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc03a69d180, {0xc0a1a17000, 0xc032f59a40, 0xc01e5d9f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0327ff6c8, {0xc0a1a17000, 0xc0371ce820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc088c05200, {0xc0a1a17000, 0xc0c22be120, 0xc01e5d9d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc03561d920) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc03561d920, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc088c05200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 69612 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc048e6cd00, 0xc0250f2960) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 67114 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab930, 0xc07f9280f0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3720 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4d00, 0xc005daf500) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3719 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c6f4820, 0xc005dafa80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2751 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00f4cd5d0, 0x14f) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc012b38360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).process(0xc00f4cd600, {0x35a0c90, 0xc0135d1a40}, 0x82b825) | |
| /app/discovery/kubernetes/pod.go:107 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1() | |
| /app/discovery/kubernetes/pod.go:98 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run | |
| /app/discovery/kubernetes/pod.go:97 +0x1d2 | |
| goroutine 2750 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc00f4cd6d0, 0x217) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x24e9980) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| k8s.io/client-go/util/workqueue.(*Type).Get(0xc012b38420) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/util/workqueue/queue.go:151 +0x9e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc00f4c1200, {0x35a0c90, 0xc0135d1ac0}, 0x82b825) | |
| /app/discovery/kubernetes/endpoints.go:154 +0x6e | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1() | |
| /app/discovery/kubernetes/endpoints.go:145 +0x53 | |
| created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run | |
| /app/discovery/kubernetes/endpoints.go:144 +0x2c5 | |
| goroutine 68733 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08e50b560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10835506 [select, 4 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc0094d9200, 0xc08221c700) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc09e9e38f0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 91611 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc051556b60, 0xc0f178e840) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68053 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0923fec60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 66953 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc10cae28f0, 0xc080698870) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 138503 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029ce5d40, 0xc07e554d80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3956623 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc07b5bba00, 0xc08d4620c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3966335 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6070, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc029aa4300, 0xc014e92900, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc029aa4300, {0xc014e92900, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc029aa4300, {0xc014e92900, 0x7d7, 0xc0652aef40}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc06e2ba568, {0xc014e92900, 0xc014e9293b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc023df6750, {0xc014e92900, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0088c7e78, {0x3544be0, 0xc023df6750}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0088c7c00, {0x7f09e11af0a0, 0xc0bf101600}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0088c7c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0088c7c00, {0xc011c11000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc04e0064e0, {0xc093551a80, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc04e0064e0}, {0xc093551a80, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc093551a80, 0x9, 0xc001380510}, {0x35449e0, 0xc04e0064e0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc093551a40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01c7a2f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc03e9f6180) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 2468 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232fc80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2448 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232fa00) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2446 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).pop(0xc01232f980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:742 +0x156 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 54107 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15940, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b0e83e80, 0xc0b1201000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b0e83e80, {0xc0b1201000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b0e83e80, {0xc0b1201000, 0xc033083040, 0xc01e5cff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0439ec888, {0xc0b1201000, 0xc036d8c4e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0648c30e0, {0xc0b1201000, 0xc0b0f32e40, 0xc01e5cfd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0afc0d200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0afc0d200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0648c30e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 2948 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 2171 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1e80}, 0xc01353f1d0, 0xc028350060) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2170 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02eccaa00) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02eccaa00, {0x35a0c90, 0xc0135d1e80}, 0xc028350060) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2169 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1e00}, 0xc01353e280, 0xc028350000) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2168 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca8c0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca8c0, {0x35a0c90, 0xc0135d1e00}, 0xc028350000) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2167 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1d80}, 0xc003d05310, 0xc027b0bf80) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2166 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca780) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca780, {0x35a0c90, 0xc0135d1d80}, 0xc027b0bf80) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2165 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1d00}, 0xc003d043c0, 0xc027b0bf20) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2164 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca640) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca640, {0x35a0c90, 0xc0135d1d00}, 0xc027b0bf20) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2163 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1c80}, 0xc01335f450, 0xc027b0bec0) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2162 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca500) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca500, {0x35a0c90, 0xc0135d1c80}, 0xc027b0bec0) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2161 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1c00}, 0xc01335e500, 0xc027b0be60) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2160 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca3c0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca3c0, {0x35a0c90, 0xc0135d1c00}, 0xc027b0be60) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2159 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1b40}, 0xc012fb5590, 0xc027b0be00) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 3766 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783d40, 0xc0169bc200) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2158 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca280) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca280, {0x35a0c90, 0xc0135d1b40}, 0xc027b0be00) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2135 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138ce540, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00e967880) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2156 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca140) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca140, {0x35a0c90, 0xc0135d1ac0}, 0xc027b0bda0) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2155 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1a40}, 0xc012d216d0, 0xc027b0bd40) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2154 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecca000) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecca000, {0x35a0c90, 0xc0135d1a40}, 0xc027b0bd40) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2153 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d19c0}, 0xc012d20780, 0xc027b0bc80) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2152 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecbbe00) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecbbe00, {0x35a0c90, 0xc0135d19c0}, 0xc027b0bc80) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2151 [select, 4 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1940}, 0xc012341810, 0xc027b0bc20) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 3804 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0202781a0, 0xc013bff500) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2150 [semacquire, 2624 minutes]: | |
| sync.runtime_Semacquire(0x0) | |
| /usr/local/go/src/runtime/sema.go:56 +0x25 | |
| sync.(*WaitGroup).Wait(0xc02ecbbcc0) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x71 | |
| github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc02ecbbcc0, {0x35a0c90, 0xc0135d1940}, 0xc027b0bc20) | |
| /app/discovery/kubernetes/kubernetes.go:586 +0x746 | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:202 +0x3b3 | |
| goroutine 2147 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc106001880, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00732e700) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2146 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc014765500, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00732e900) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2145 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01401c380, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00732eb00) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3702 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c7825b0, 0xc016aac280) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 67044 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0234ab930, 0xc0678dab10) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3695 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c7828f0, 0xc016aad180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2175 [select, 1 minutes]: | |
| github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).updater(0xc001057cc0, {0x35a0c90, 0xc0135d1f80}, 0xc0136eb090, 0xc028350180) | |
| /app/discovery/legacymanager/manager.go:208 +0xdf | |
| created by github.com/prometheus/prometheus/discovery/legacymanager.(*Manager).startProvider | |
| /app/discovery/legacymanager/manager.go:203 +0x452 | |
| goroutine 2142 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01401c0e0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00d717b80) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3675 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0197c1040, 0xc017164d80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3782 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf520, 0xc015845900) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 2575 [select, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:373 +0x11a | |
| created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:367 +0x2e5 | |
| goroutine 10852205 [select]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc01fa8b680, 0xc0f9a22f00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0d6666570) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 2140 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01398d7a0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00eb1c400) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2131 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc013ffa9a0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00eec3600) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2202 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00cfc9180, 0x0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:628 +0x45 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2337 [chan receive, 4 minutes]: | |
| k8s.io/client-go/tools/cache.(*processorListener).run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:772 +0x49 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f09ec335e98) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00ccca738, {0x354b380, 0xc020724ab0}, 0x1, 0xc02790a720) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00cab33b0, 0x3b9aca00, 0x0, 0xa0, 0xc00ccca788) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:133 +0x89 | |
| k8s.io/apimachinery/pkg/util/wait.Until(...) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:90 | |
| k8s.io/client-go/tools/cache.(*processorListener).run(0xc00f4c1080) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/shared_informer.go:771 +0x6b | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2203 [chan receive, 2624 minutes]: | |
| k8s.io/client-go/tools/cache.(*controller).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:130 +0x28 | |
| created by k8s.io/client-go/tools/cache.(*controller).Run | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/controller.go:129 +0x105 | |
| goroutine 2133 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc106001420, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00eec2080) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2132 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc014765420, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00eec3000) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2204 [select]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc01401cc40, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc040750440}, 0xc003ad9ce8, 0xc013f17a40, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc01401cc40, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc012c312c0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc019e04c40, {0x354b360, 0xc012c31220}, 0x1, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc01401cc40, 0xc02749a300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2134 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc01392eee0, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00e967e80) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 2350 [select, 1 minutes]: | |
| k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc014765960, {0x0, 0x0, 0x4d8e460}, {0x357ade0, 0xc053739ec0}, 0xc101931ce8, 0xc014ecad20, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:468 +0x1b6 | |
| k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc014765960, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:428 +0x5f6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run.func1() | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:221 +0x26 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc013035770) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:155 +0x67 | |
| k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0179d6800, {0x354b360, 0xc0130356d0}, 0x1, 0xc028350360) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:156 +0xb6 | |
| k8s.io/client-go/tools/cache.(*Reflector).Run(0xc014765960, 0xc028350360) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/tools/cache/reflector.go:220 +0x13d | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:56 +0x22 | |
| k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:73 +0x5a | |
| created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/wait/wait.go:71 +0x88 | |
| goroutine 2141 [select]: | |
| github.com/prometheus/prometheus/rules.(*Group).run(0xc0138ceb60, {0x35a0cc8, 0xc000056030}) | |
| /app/rules/manager.go:415 +0x8b5 | |
| github.com/prometheus/prometheus/rules.(*Manager).Update.func1(0xc00d717f80) | |
| /app/rules/manager.go:999 +0xaf | |
| created by github.com/prometheus/prometheus/rules.(*Manager).Update | |
| /app/rules/manager.go:989 +0x3b2 | |
| goroutine 3673 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0197c0dd0, 0xc017276180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3796 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcf860, 0xc013cc0780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 97808 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02602b9e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3612 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcef70, 0x0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 146162 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5fe78, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc111f1e680, 0xc0fe05d200, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc111f1e680, {0xc0fe05d200, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc111f1e680, {0xc0fe05d200, 0x7d7, 0xc116bd4ca0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0144775c0, {0xc0fe05d200, 0xc0fe05d23b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc025849b00, {0xc0fe05d200, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc1039bc5f8, {0x3544be0, 0xc025849b00}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc1039bc380, {0x7f09e11af0a0, 0xc108fe3180}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc1039bc380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc1039bc380, {0xc01b22f000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0f8dacf60, {0xc01d528900, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0f8dacf60}, {0xc01d528900, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc01d528900, 0x9, 0xc001269350}, {0x35449e0, 0xc0f8dacf60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc01d5288c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0104f6f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0f0644300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3707 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c782ea0, 0xc0169bd300) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852569 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 10823067 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08957db00, 0xc0a31d9d00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc002161200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 3798 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01cfcfad0, 0xc013cc0780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3873 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279520, 0xc01512f580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3742 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0203591e0, 0xc013cc0780) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10853072 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3588, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04805a380, 0xc096ed4791, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc04805a380, {0xc096ed4791, 0x1, 0x1}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc04805a380, {0xc096ed4791, 0x3, 0x28a2620}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc06c20ebe0, {0xc096ed4791, 0x405780, 0xc003df5720}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).backgroundRead(0xc096ed4780) | |
| /usr/local/go/src/net/http/server.go:672 +0x3f | |
| created by net/http.(*connReader).startBackgroundRead | |
| /usr/local/go/src/net/http/server.go:668 +0xcf | |
| goroutine 10823930 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc033858c48, 0x6) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc033858c30, {0xc09a00e601, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc09a00e601, 0xc070e13d10, 0xc070e13cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0bf370500) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0bf370500) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0bf370500, {0x2748340, 0xc09e3f2210}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06abd8e70, {0xc018a6bc00, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc025097450, 0xc045dc6090, {0x357ac00, 0xc097a8bcc0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc08c4f10a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc04eb3e140) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 10851853 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0866ca120) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3856 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020358340, 0xc016dffc80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3894 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020278d00, 0xc013bfec00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3895 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279110, 0xc013bff080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3896 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020279380, 0xc013bff500) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3864 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020359930, 0xc016f97400) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10848422 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0487976c8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc02074fba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0487976b0, {0xc03bb2e001, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc03bb2e001, 0xc0a4869600, 0xc02074fcf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc08969a280) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc08969a280) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc08969a280, {0x2748340, 0xc0255fe720}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08d311e00, {0xc08db60000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0489f1590, 0xc056c946c0, {0x357ac00, 0xc0255e6340}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0489ecfc0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc048a17700) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 3966257 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b01afa00, 0xc074763620) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 9719118 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0c08b9ba0, 0xc05df8a440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10828888 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc08db324c8, 0x5) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08db324b0, {0xc0fe508001, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x5101}, {0xc0fe508001, 0x416f26, 0xc01e123cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc08db508c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc08db508c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc08db508c0, {0x2748340, 0xc037bf2ee8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08df14de0, {0xc08db61000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc08df2e0f0, 0xc0180e8480, {0x357ac00, 0xc0a2c34400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc08db01d60) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc08d6df300) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 9712351 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0731500, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc045e99900, 0xc0b1e7c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc045e99900, {0xc0b1e7c000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc045e99900, {0xc0b1e7c000, 0xc032f58780, 0xc0229a7f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0e00c6028, {0xc0b1e7c000, 0xc042a01040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08f027680, {0xc0b1e7c000, 0xc09ec3a240, 0xc0229a7d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04149e480) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04149e480, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08f027680) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3904 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a340, 0xc016f52f00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 93101 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03de71680) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10849914 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc098590180, 0xc097a3df00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc09861ee40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 3907 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a0d0, 0xc00ce502c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 101674 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd0a90, 0xc0892919c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 9719039 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029f5bee0, 0xc0530f9440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3910 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a5b0, 0xc00cc879c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3911 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a750, 0xc00cc876c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3912 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9aea0, 0x0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3913 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9af70, 0xc005786a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3915 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9a270, 0xc005649c80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 132746 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029909380, 0xc0eeb5fe80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3918 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9b5f0, 0xc005786400) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3920 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020d9ab60, 0xc013bff080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3830 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04e6910e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3971 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0578218c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4894 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18ff440, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc063f84080, 0xc0640aa000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc063f84080, {0xc0640aa000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc063f84080, {0xc0640aa000, 0xc032f58780, 0xc020e12f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0176f8fa8, {0xc0640aa000, 0xc02efd6d00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc027efa7e0, {0xc0640aa000, 0xc0679d7440, 0xc020e12d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc05fd76480) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc05fd76480, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc027efa7e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5173 [select]: | |
| net/http.(*persistConn).writeLoop(0xc036509e60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4033 [select]: | |
| net/http.(*persistConn).writeLoop(0xc057821c20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67112 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab6c0, 0xc06b5651a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5417 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24c60, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0992e8080, 0xc099300000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0992e8080, {0xc099300000, 0x5867, 0x5867}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0992e8080, {0xc099300000, 0x5862, 0xc096f142e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0179c0030, {0xc099300000, 0xc099300000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc022048d50, {0xc099300000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc076d03778, {0x3544be0, 0xc022048d50}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc076d03500, {0x7f09e11af0a0, 0xc096db5980}, 0x5867) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc076d03500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc076d03500, {0xc096f2b000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc096f26360, {0xc02c9e1a80, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc096f26360}, {0xc02c9e1a80, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02c9e1a80, 0x9, 0xc001dea000}, {0x35449e0, 0xc096f26360}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02c9e1a40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc02084af98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc096f28000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 93100 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24f18, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0c4426180, 0xc0c56f6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0c4426180, {0xc0c56f6000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0c4426180, {0xc0c56f6000, 0xc032f53a40, 0xc01dd70f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc03451b7e8, {0xc0c56f6000, 0xc07cb46340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03de71680, {0xc0c56f6000, 0xc0f129cf60, 0xc01dd70d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0c44b6de0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0c44b6de0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03de71680) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5522 [select]: | |
| net/http.(*persistConn).writeLoop(0xc049d0f8c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 66951 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc10cae2820, 0xc0bcf26380) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5240 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d25488, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc079daef00, 0xc07ba30000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc079daef00, {0xc07ba30000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc079daef00, {0xc07ba30000, 0xc032f59b80, 0xc029c3bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01e632a48, {0xc07ba30000, 0xc0140a69c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0368ead80, {0xc07ba30000, 0xc0306037a0, 0xc029c3bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc076aaf920) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc076aaf920, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0368ead80) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3966200 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd0c30, 0xc0c8e2c150) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10850358 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc08bc753c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc08bc753b8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08bc753b0, {0xc05cca9400, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc05cca9400, 0x217c472, 0xc003df5cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05d705180) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05d705180) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05d705180, {0x2748340, 0xc092784450}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0b107f3b0, {0xc070466c00, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0c9174460, 0xc003df5e38, {0x357ac00, 0xc03dbf9b40}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0881c89e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc03dbf9b00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 10852723 [select]: | |
| net/http.(*persistConn).writeLoop(0xc062ee4ea0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5106 [select]: | |
| net/http.(*persistConn).writeLoop(0xc023ba3b00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67116 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228abad0, 0xc07f9280f0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5039 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18feed0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06f5d1b00, 0xc070538000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06f5d1b00, {0xc070538000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06f5d1b00, {0xc070538000, 0xc032c9ba40, 0xc01b517f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01a2daf40, {0xc070538000, 0xc01b3824e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0316959e0, {0xc070538000, 0xc0573f3680, 0xc01b517d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc06b0c7920) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc06b0c7920, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0316959e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 68216 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0b5e44ea0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10841821 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc02cdea480, 0xc0987ea400) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc087dfce40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4877 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b48b0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05fd03100, 0xc05fd50000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05fd03100, {0xc05fd50000, 0x3c72, 0x3c72}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05fd03100, {0xc05fd50000, 0x3c6d, 0xc05e96d520}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01e80a160, {0xc05fd50000, 0xc05fd50000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc079172630, {0xc05fd50000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc04e5ebaf8, {0x3544be0, 0xc079172630}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc04e5eb880, {0x7f09e11af0a0, 0xc05ed8a500}, 0x3c72) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc04e5eb880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc04e5eb880, {0xc05ede4000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc05edae720, {0xc029bfb700, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc05edae720}, {0xc029bfb700, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc029bfb700, 0x9, 0xc001bfa150}, {0x35449e0, 0xc05edae720}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc029bfb6c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc105feff98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc05e93ac00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6073 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b4998, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc023a20000, 0xc0a519a400, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc023a20000, {0xc0a519a400, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc023a20000, {0xc0a519a400, 0x7b5, 0xc0038f1860}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01499e980, {0xc0a519a400, 0xc0a519a45d, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01e115dd0, {0xc0a519a400, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a2979af8, {0x3544be0, 0xc01e115dd0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a2979880, {0x7f09e11af0a0, 0xc003985e00}, 0x7ba) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a2979880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a2979880, {0xc005fba000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0039abe60, {0xc0316cf380, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0039abe60}, {0xc0316cf380, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0316cf380, 0x9, 0xc0019b09c0}, {0x35449e0, 0xc0039abe60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0316cf340) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc02080df98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a92d1c80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 9712265 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0353d3380, 0xc0594dc080) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4892 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18ff528, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05fd03700, 0xc05fd72000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05fd03700, {0xc05fd72000, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05fd03700, {0xc05fd72000, 0x7d7, 0xc05fd454e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0176f8ee8, {0xc05fd72000, 0xc05fd7203b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc079173500, {0xc05fd72000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc05e5cfaf8, {0x3544be0, 0xc079173500}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc05e5cf880, {0x7f09e11af0a0, 0xc05fd48fc0}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc05e5cf880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc05e5cf880, {0xc05fd7a000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc05fd763c0, {0xc02a028ba0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc05fd763c0}, {0xc02a028ba0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02a028ba0, 0x9, 0xc0014498c0}, {0x35449e0, 0xc05fd763c0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02a028b60) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003a67f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc05ee66600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10852812 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 4611 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b5560, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04e5fd600, 0xc04ae87000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc04e5fd600, {0xc04ae87000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc04e5fd600, {0xc04ae87000, 0xc032f592c0, 0xc01e5e1f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c8771a8, {0xc04ae87000, 0xc0136dfba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0567659e0, {0xc04ae87000, 0xc02509f5c0, 0xc01e5e1d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04ae90c60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04ae90c60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0567659e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852807 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fed00, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc025094d00, 0xc0877ebaa0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc025094d00, {0x35a0d00, 0xc02d73bb00}, {0xc01e11d2f8, 0x40ce34}, {0x354bea0, 0xc07f6b28c0}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc025094d00, {0x35a0d00, 0xc02d73bb00}, {0x35c3098, 0x0}, {0x35c3098, 0xc0877eba70}, 0xc01e11d4e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0877eba70}, 0xc085ef5fd0, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc025094c80, {0x35a0d00, 0xc02d73bb00}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc02d73bb00, {0x35a0d00, 0xc02d73bb00}, 0x0, 0xc023544ad6) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc025094c80, {0x35a0d00, 0xc02d73bb00}, {0x357b3f8, 0xc0877eba70}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc025094c80, {0x35a0d00, 0xc02d73bb00}, {0xc02d1f55a0, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc0197b7500, {0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0xc02d1f5580}, {0xc023544b28, 0xc10197e790}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0x3}, {0xc023544b28, 0x13}, {0xc017c696e0, 0x0}, 0xc0197b2ec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0x3}, {0xc023544b28, 0x13}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc02d73bb00, {0x35a0d00, 0xc02d73bb00}, {0x2c655e4, 0x0}, {0xc023544b28, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f59e00, {0x35a0d00, 0xc02d73bb00}, {{}, 0x0, {0xc043cae4c0, 0x5}, {0xc023544b28, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x82b7a6, 0xc0ecafc790) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 4443 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3db0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc044598380, 0xc0479f2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc044598380, {0xc0479f2000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc044598380, {0xc0479f2000, 0xc032f52f00, 0xc01bba4f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b055f30, {0xc0479f2000, 0xc01b383040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02dfed560, {0xc0479f2000, 0xc04e1e7ce0, 0xc01bba4d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0479ee000) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0479ee000, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02dfed560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10762829 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5fad8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0db5b7a80, 0xc0995a5000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0db5b7a80, {0xc0995a5000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0db5b7a80, {0xc0995a5000, 0xc032f59b80, 0xc0a6260f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc045807db0, {0xc0995a5000, 0xc040bd11e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0632b9b00, {0xc0995a5000, 0xc0a4287260, 0xc0a6260d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc05fbd7980) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc05fbd7980, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0632b9b00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852748 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 92394 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fe960, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a2100680, 0xc0a27f0000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a2100680, {0xc0a27f0000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a2100680, {0xc0a27f0000, 0xc032f53540, 0xc04aa1af68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc053c596b0, {0xc0a27f0000, 0xc019d5f6c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0154bfb00, {0xc0a27f0000, 0xc0afed2a80, 0xc04aa1ad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a0f74ea0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a0f74ea0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0154bfb00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5893 [select]: | |
| net/http.(*persistConn).writeLoop(0xc051bb7440) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10834102 [select, 4 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc089237e00, 0xc08009ba00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc0adacf320) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 6359 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732e40, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0163ae900, 0xc0cd37a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0163ae900, {0xc0cd37a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0163ae900, {0xc0cd37a000, 0xc032f58780, 0xc0153e6f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0135adcf8, {0xc0cd37a000, 0xc02f01b520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0315cc6c0, {0xc0cd37a000, 0xc029e9ac00, 0xc0153e6d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc012fd9560) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc012fd9560, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0315cc6c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5886 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b950, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a11d8280, 0xc0a38b3000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a11d8280, {0xc0a38b3000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a11d8280, {0xc0a38b3000, 0xc032f592c0, 0xc020853f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013faf118, {0xc0a38b3000, 0xc028be44e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0505d1e60, {0xc0a38b3000, 0xc08792c0c0, 0xc020853d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a2e79200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a2e79200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0505d1e60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5846 [select]: | |
| net/http.(*persistConn).writeLoop(0xc051bb7320) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 68051 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0923fe7e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10848195 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6410, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05dbe3f00, 0xc0118ff000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05dbe3f00, {0xc0118ff000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05dbe3f00, {0xc0118ff000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc02afe0260, {0xc0118ff000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc015d90360, {0xc0118ff000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc07d8af6e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc07d8af6e0, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc044be6320, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 5346 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03af04000) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5345 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d250e8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc080877f80, 0xc07eec5000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc080877f80, {0xc07eec5000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc080877f80, {0xc07eec5000, 0xc033082f00, 0xc020855f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b668c10, {0xc07eec5000, 0xc01448fa00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03af04000, {0xc07eec5000, 0xc060bbcfc0, 0xc020855d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc087406540) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc087406540, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03af04000) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5577 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2ae70, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09b097200, 0xc09bb5c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09b097200, {0xc09bb5c000, 0x4cb1, 0x4cb1}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09b097200, {0xc09bb5c000, 0x4cac, 0xc09ba99fc0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017b06258, {0xc09bb5c000, 0xc09bb5c000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0242e0990, {0xc09bb5c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc08b8cd778, {0x3544be0, 0xc0242e0990}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc08b8cd500, {0x7f09e11af0a0, 0xc09baa2e80}, 0x4cb1) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc08b8cd500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc08b8cd500, {0xc09bb4e000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09bb48c00, {0xc02ca6c660, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09bb48c00}, {0xc02ca6c660, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02ca6c660, 0x9, 0xc0011e6090}, {0x35449e0, 0xc09bb48c00}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02ca6c620) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003b54f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc08b8a2600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 68050 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24d48, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0e44fff00, 0xc0ebe56000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0e44fff00, {0xc0ebe56000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0e44fff00, {0xc0ebe56000, 0xc032f53180, 0xc01c7aff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04fe562d0, {0xc0ebe56000, 0xc053e6e1a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0923fe7e0, {0xc0ebe56000, 0xc0dba9ef00, 0xc01c7afd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0e547ac60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0e547ac60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0923fe7e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10849410 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08eb12d80, 0xc08ead4d00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce8c6, 0xc0dba1bb00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4179 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f650, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc024e32c00, 0xc04cb6a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc024e32c00, {0xc04cb6a000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc024e32c00, {0xc04cb6a000, 0x3c6f, 0xc024e27de0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b3fd398, {0xc04cb6a000, 0xc04cb6a000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc025725620, {0xc04cb6a000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc024d86cf8, {0x3544be0, 0xc025725620}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc024d86a80, {0x7f09e11af0a0, 0xc024c9de00}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc024d86a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc024d86a80, {0xc024e72000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc024e437a0, {0xc017200040, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc024e437a0}, {0xc017200040, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc017200040, 0x9, 0xc001689b90}, {0x35449e0, 0xc024e437a0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc017200000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0229a4f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0512f1380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4109 [select]: | |
| net/http.(*persistConn).writeLoop(0xc01be6ac60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10848188 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2a9e8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05dbe3d80, 0xc053466000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05dbe3d80, {0xc053466000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05dbe3d80, {0xc053466000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0d71a0968, {0xc053466000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc052af4210, {0xc053466000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc00f2c3080) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc00f2c3080, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc047005220, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 69225 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a64f8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc033712580, 0xc010652000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc033712580, {0xc010652000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc033712580, {0xc010652000, 0xc033086000, 0xc0713c6f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0436053e0, {0xc010652000, 0xc02f7a29c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0576bc000, {0xc010652000, 0xc0baac0900, 0xc0713c6d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc016b28ea0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc016b28ea0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0576bc000) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4105 [select]: | |
| net/http.(*persistConn).writeLoop(0xc00baa45a0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4129 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f908, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc021f9f500, 0xc08538c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc021f9f500, {0xc08538c000, 0x3c70, 0x3c70}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc021f9f500, {0xc08538c000, 0x3c6b, 0xc022c27340}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b0d6d10, {0xc08538c000, 0xc08538c000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc025715488, {0xc08538c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc022c72278, {0x3544be0, 0xc025715488}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc022c72000, {0x7f09e11af0a0, 0xc022c37a80}, 0x3c70) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc022c72000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc022c72000, {0xc022c76000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc022c74c60, {0xc015d25e00, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc022c74c60}, {0xc015d25e00, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc015d25e00, 0x9, 0xc001688690}, {0x35449e0, 0xc022c74c60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc015d25dc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc020459f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0354b5800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4104 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d165f0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc021f9f600, 0xc022a99000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc021f9f600, {0xc022a99000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc021f9f600, {0xc022a99000, 0xc032f59b80, 0xc02083cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b3fc420, {0xc022a99000, 0xc0148fa9c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc00baa45a0, {0xc022a99000, 0xc0382fa480, 0xc02083cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc022a94ea0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc022a94ea0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc00baa45a0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4039 [select]: | |
| net/http.(*persistConn).writeLoop(0xc010410900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10843399 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc05d5c8180, 0xc05cee7300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce8c6, 0xc062fc10e0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4047 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f568, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02354b000, 0xc023a04000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02354b000, {0xc023a04000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02354b000, {0xc023a04000, 0x3c6f, 0xc023556660}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b20c868, {0xc023a04000, 0xc023a04000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc025721e00, {0xc023a04000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc023385078, {0x3544be0, 0xc025721e00}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc023384e00, {0x7f09e11af0a0, 0xc02354c600}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc023384e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc023384e00, {0xc02357a000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc023571ec0, {0xc0165323c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc023571ec0}, {0xc0165323c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0165323c0, 0x9, 0xc0016892f0}, {0x35449e0, 0xc023571ec0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc016532380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc020e10f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc042f74a80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5723 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c430, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09f207c00, 0xc0a1186000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09f207c00, {0xc0a1186000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09f207c00, {0xc0a1186000, 0xc032f523c0, 0xc01c7baf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017b06c68, {0xc0a1186000, 0xc0137e8340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04f2edb00, {0xc0a1186000, 0xc032e23e00, 0xc01c7bad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a0a41d40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a0a41d40, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04f2edb00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 93708 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6240, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0df808980, 0xc0ded9c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0df808980, {0xc0ded9c000, 0x17d7, 0x17d7}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0df808980, {0xc0ded9c000, 0x1793, 0xc0df8324a0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc038bc4030, {0xc0ded9c000, 0xc0ded9c03f, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0255883a8, {0xc0ded9c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0ddbfb778, {0x3544be0, 0xc0255883a8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0ddbfb500, {0x7f09e11af0a0, 0xc0df303640}, 0x1798) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0ddbfb500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0ddbfb500, {0xc0df876000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0df872360, {0xc04eea77e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0df872360}, {0xc04eea77e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc04eea77e0, 0x9, 0xc0011d2ea0}, {0x35449e0, 0xc0df872360}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc04eea77a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003ae8f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0df83d200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5105 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fe878, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc071a3e100, 0xc073214000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc071a3e100, {0xc073214000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc071a3e100, {0xc073214000, 0xc032f537c0, 0xc01a13af68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b6687c8, {0xc073214000, 0xc01596a000, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc023ba3b00, {0xc073214000, 0xc01f7dec60, 0xc01a13ad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc072d0b680) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc072d0b680, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc023ba3b00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 91634 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a92815f0, 0xc038917580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5172 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fe6a8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc076d4cc00, 0xc077cf6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc076d4cc00, {0xc077cf6000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc076d4cc00, {0xc077cf6000, 0xc032f59b80, 0xc0229a5f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01a2db058, {0xc077cf6000, 0xc0178ced00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc036509e60, {0xc077cf6000, 0xc0326c7560, 0xc0229a5d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc077ce6540) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc077ce6540, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc036509e60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 69249 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3e98, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a37ed980, 0xc00b67a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a37ed980, {0xc00b67a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a37ed980, {0xc00b67a000, 0xc032f53680, 0xc01e612f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc044072130, {0xc00b67a000, 0xc0f23af1e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0442a3320, {0xc00b67a000, 0xc0c4805560, 0xc01e612d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc013589140) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc013589140, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0442a3320) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 9709935 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc067ede680, 0xc059121e00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 9823682 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0731160, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0dfb40e00, 0xc03b3b1000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0dfb40e00, {0xc03b3b1000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0dfb40e00, {0xc03b3b1000, 0xc032f532c0, 0xc01c7adf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0498830f8, {0xc03b3b1000, 0xc02c49a9c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc068545560, {0xc03b3b1000, 0xc0b5f499e0, 0xc01c7add30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc029f6a360) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc029f6a360, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc068545560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10825442 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc02b020780, 0xc08316fc00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0d9786180) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 10852551 [select]: | |
| net/http.(*persistConn).writeLoop(0xc00f3959e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 90850 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02939d040, 0xc03a3b4800) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 94468 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0220be410, 0xc0abebd440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10850646 [select]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc0400ac300, 0xc0a5a89200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc065556fc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4232 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d16250, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0265b3380, 0xc029e90000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0265b3380, {0xc029e90000, 0x4c50, 0x4c50}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0265b3380, {0xc029e90000, 0x4c4b, 0xc029a40b80}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c06fd38, {0xc029e90000, 0xc029e90000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0255e11a0, {0xc029e90000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc024edde78, {0x3544be0, 0xc0255e11a0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc024eddc00, {0x7f09e11af0a0, 0xc029a74040}, 0x4c50) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc024eddc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc024eddc00, {0xc029a97000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc029a858c0, {0xc017de77e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc029a858c0}, {0xc017de77e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc017de77e0, 0x9, 0xc001381200}, {0x35449e0, 0xc029a858c0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc017de77a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc105feef98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02d3a2c00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10851826 [select]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc01df10000, 0xc0aada6b00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x82b7a6, 0xc0e90a56b0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4257 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15dc8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02a69e880, 0xc02a7ac000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02a69e880, {0xc02a7ac000, 0x3e9f, 0x3e9f}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02a69e880, {0xc02a7ac000, 0x3e9a, 0xc029a40d80}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c06fd60, {0xc02a7ac000, 0xc02a7ac000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0258905d0, {0xc02a7ac000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc029aba278, {0x3544be0, 0xc0258905d0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc029aba000, {0x7f09e11af0a0, 0xc029a74480}, 0x3e9f) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc029aba000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc029aba000, {0xc029acb000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc029abfc80, {0xc017de7a80, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc029abfc80}, {0xc017de7a80, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc017de7a80, 0x9, 0xc001b48f00}, {0x35449e0, 0xc029abfc80}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc017de7a40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01ac78f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02d3a3080) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10848196 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c178, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05dbe3f80, 0xc0b2f67000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05dbe3f80, {0xc0b2f67000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05dbe3f80, {0xc0b2f67000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc02afe0268, {0xc0b2f67000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc015d90300, {0xc0b2f67000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc090e86420) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc090e86420, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc044be63c0, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 4248 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15ce0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02a303d80, 0xc029e95000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02a303d80, {0xc029e95000, 0x4cb1, 0x4cb1}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02a303d80, {0xc029e95000, 0x19fb, 0xc0299a9c60}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b3fddc0, {0xc029e95000, 0xc029e982b1, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc025636cd8, {0xc029e95000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0272e2cf8, {0x3544be0, 0xc025636cd8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0272e2a80, {0x7f09e11af0a0, 0xc0299b2f40}, 0x1a00) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0272e2a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0272e2a80, {0xc0299e8000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0299bd020, {0xc017c6fd20, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0299bd020}, {0xc017c6fd20, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc017c6fd20, 0x9, 0xc001852780}, {0x35449e0, 0xc0299bd020}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc017c6fce0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0229a1f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc030a8bc80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5825 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c090, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09c17f780, 0xc0a2102000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09c17f780, {0xc0a2102000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09c17f780, {0xc0a2102000, 0xc033082f00, 0xc0209b4f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00ef7b2c0, {0xc0a2102000, 0xc0136df040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04d3a7680, {0xc0a2102000, 0xc08544c7e0, 0xc0209b4d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc09f1f9a40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc09f1f9a40, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04d3a7680) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5910 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d252b8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a11d8400, 0xc016bda000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a11d8400, {0xc016bda000, 0x3c72, 0x3c72}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a11d8400, {0xc016bda000, 0x3c6d, 0xc0a215dc80}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00ef7b3f0, {0xc016bda000, 0xc016bda000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01e03d800, {0xc016bda000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a12753f8, {0x3544be0, 0xc01e03d800}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a1275180, {0x7f09e11af0a0, 0xc0a2533400}, 0x3c72) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a1275180, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a1275180, {0xc0a2567000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a255acc0, {0xc02ea652a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a255acc0}, {0xc02ea652a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02ea652a0, 0x9, 0xc0012e9bc0}, {0x35449e0, 0xc0a255acc0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02ea65260) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc020dc7f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a1f7cc00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 3966421 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b4e20, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc03144fa00, 0xc008c70000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc03144fa00, {0xc008c70000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc03144fa00, {0xc008c70000, 0x3ccb, 0xc04547c2c0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01e8d1a88, {0xc008c70000, 0xc008c70000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0255d9a10, {0xc008c70000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc00820ecf8, {0x3544be0, 0xc0255d9a10}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc00820ea80, {0x7f09e11af0a0, 0xc09100a700}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc00820ea80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc00820ea80, {0xc0a2549000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc003adb380, {0xc00900f9a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc003adb380}, {0xc00900f9a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00900f9a0, 0x9, 0xc001380c90}, {0x35449e0, 0xc003adb380}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00900f960) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01cac8f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc030556300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10852641 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605070, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b3786580, 0xc0b46baa20, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc0b3786580, {0x35a0d00, 0xc0254f91a0}, {0xc008d412f8, 0x40ce34}, {0x354bea0, 0xc0b323fb00}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc0b3786580, {0x35a0d00, 0xc0254f91a0}, {0x35c3098, 0x0}, {0x35c3098, 0xc0b46ba9f0}, 0xc008d414e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0b46ba9f0}, 0xc0bd099eb0, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc0b3786500, {0x35a0d00, 0xc0254f91a0}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc0254f91a0, {0x35a0d00, 0xc0254f91a0}, 0xc01d8c95a8, 0x0) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc0b3786500, {0x35a0d00, 0xc0254f91a0}, {0x357b3f8, 0xc0b46ba9f0}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc0b3786500, {0x35a0d00, 0xc0254f91a0}, {0xc02560e690, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc019784a80, {0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0xc02560e670}, {0xc00de2ab28, 0xc024d8b4a0}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0x3}, {0xc00de2ab28, 0x12}, {0xc017c69860, 0x0}, 0xc019563e40) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0x3}, {0xc00de2ab28, 0x12}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc0254f91a0, {0x35a0d00, 0xc0254f91a0}, {0x2c655e4, 0x1}, {0xc00de2ab28, 0x23}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f52280, {0x35a0d00, 0xc0254f91a0}, {{}, 0x0, {0xc032cb0080, 0x5}, {0xc00de2ab28, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x354c040, 0xc0063c6840) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 10843431 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc05d576900, 0xc05d56b100) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce8c6, 0xc048cf4b40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4303 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05eeebe60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 68698 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08e4fa480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4462 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b31e8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04ad2ec00, 0xc08e5b2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc04ad2ec00, {0xc08e5b2000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc04ad2ec00, {0xc08e5b2000, 0x3ccb, 0xc04be69d20}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c177eb0, {0xc08e5b2000, 0xc08e5b2000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc039ab6870, {0xc08e5b2000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc043aa1e78, {0x3544be0, 0xc039ab6870}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc043aa1c00, {0x7f09e11af0a0, 0xc04bf47440}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc043aa1c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc043aa1c00, {0xc04bfbb000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc04bfb0600, {0xc0194bc3c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc04bfb0600}, {0xc0194bc3c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0194bc3c0, 0x9, 0xc001a73710}, {0x35449e0, 0xc04bfb0600}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0194bc380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc029c37f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc04bf12d80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4590 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b52a8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc04ad2ed00, 0xc04fefe000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc04ad2ed00, {0xc04fefe000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc04ad2ed00, {0xc04fefe000, 0xc032f53540, 0xc01a138f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01d81f688, {0xc04fefe000, 0xc028be4000, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc053f87e60, {0xc04fefe000, 0xc054eb3860, 0xc01a138d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04edf8a20) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04edf8a20, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc053f87e60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 68439 [select]: | |
| net/http.(*persistConn).writeLoop(0xc077644480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67662 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd0410, 0xc0252ab340) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10825405 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc02983b980, 0xc0aabea000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a16a0, 0xc016f50580) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 4531 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03f76c6c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4530 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3a10, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0487df900, 0xc0488d8000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0487df900, {0xc0488d8000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0487df900, {0xc0488d8000, 0x4486af, 0xc02e507ec8}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c876ed8, {0xc0488d8000, 0x0, 0x2}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03f76c6c0, {0xc0488d8000, 0x448c40, 0xc02e507ec8}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0488d6000) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0488d6000, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03f76c6c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 66448 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0039b51e0, 0xc07b21c0c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4512 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3928, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0483e2180, 0xc0488c2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0483e2180, {0xc0488c2000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0483e2180, {0xc0488c2000, 0x4486af, 0xc003bcfec8}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c876ed0, {0xc0488c2000, 0x0, 0x2}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0704e9e60, {0xc0488c2000, 0x448c40, 0xc003bcfec8}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc048341ec0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc048341ec0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0704e9e60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4778405 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0050c18c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5023 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fea48, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0729a7c00, 0xc07628e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0729a7c00, {0xc07628e000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0729a7c00, {0xc07628e000, 0x4c50, 0xc072d04180}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01b6687a0, {0xc07628e000, 0xc07628e000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020c6edc8, {0xc07628e000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc07267ecf8, {0x3544be0, 0xc020c6edc8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc07267ea80, {0x7f09e11af0a0, 0xc072c76ac0}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc07267ea80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc07267ea80, {0xc072d0c000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc072d0a600, {0xc02bc892a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc072d0a600}, {0xc02bc892a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02bc892a0, 0x9, 0xc001591aa0}, {0x35449e0, 0xc072d0a600}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02bc89260) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0713c8f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc06ed75980) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4398 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3af8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc047d22900, 0xc047fad000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc047d22900, {0xc047fad000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc047d22900, {0xc047fad000, 0x4486af, 0xc01cb54ec8}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01cd32b38, {0xc047fad000, 0x0, 0x2}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03e399200, {0xc047fad000, 0x448c40, 0xc01cb54ec8}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc04880e6c0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc04880e6c0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03e399200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4529 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0704e9e60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 68215 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bac10, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0fd249a80, 0xc108707000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0fd249a80, {0xc108707000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0fd249a80, {0xc108707000, 0xc032f59cc0, 0xc01cb59f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc05029f030, {0xc108707000, 0xc02a3004e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0b5e44ea0, {0xc108707000, 0xc108db1f80, 0xc01cb59d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc108dbd380) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc108dbd380, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0b5e44ea0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 93288 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc052d052b0, 0xc021206f00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 93291 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5ff60, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0c8d7f880, 0xc0c8053000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0c8d7f880, {0xc0c8053000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0c8d7f880, {0xc0c8053000, 0xc032f53040, 0xc003b41f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc037a04228, {0xc0c8053000, 0xc040bd1380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05e3ef440, {0xc0c8053000, 0xc0d87de4e0, 0xc003b41d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0c97c3320) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0c97c3320, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05e3ef440) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 91639 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb6f0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0f272f200, 0xc014324c00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0f272f200, {0xc014324c00, 0xbca, 0xbca}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0f272f200, {0xc014324c00, 0xb8a, 0xc06ea1ab20}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04bb15700, {0xc014324c00, 0xc014324c3b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc039aeef18, {0xc014324c00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc06f3c0278, {0x3544be0, 0xc039aeef18}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc06f3c0000, {0x7f09e11af0a0, 0xc06f3bc380}, 0xb8f) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc06f3c0000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc06f3c0000, {0xc08ee45000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc06f3c2120, {0xc0421c8f20, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc06f3c2120}, {0xc0421c8f20, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0421c8f20, 0x9, 0xc001a73d70}, {0x35449e0, 0xc06f3c2120}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0421c8ee0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc013a26f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc064477380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10603131 [select]: | |
| net/http.(*persistConn).writeLoop(0xc126c8e900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5826 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04d3a7680) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5726 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04eee98c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5241 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0368ead80) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67235 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f738, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0befa6580, 0xc0bf10a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0befa6580, {0xc0bf10a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0befa6580, {0xc0bf10a000, 0xc032f523c0, 0xc01a139f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04cbd0200, {0xc0bf10a000, 0xc0280c5380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06ef14a20, {0xc0bf10a000, 0xc0be9845a0, 0xc01a139d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0bf03c6c0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0bf03c6c0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06ef14a20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67186 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06c9f6360) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852642 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 4591 [select]: | |
| net/http.(*persistConn).writeLoop(0xc053f87e60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4649 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b51c0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0500c2500, 0xc0841aa000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0500c2500, {0xc0841aa000, 0x3c70, 0x3c70}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0500c2500, {0xc0841aa000, 0x3c6b, 0xc0520eb540}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01d81f720, {0xc0841aa000, 0xc0841aa000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc040421620, {0xc0841aa000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc04e773778, {0x3544be0, 0xc040421620}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc04e773500, {0x7f09e11af0a0, 0xc0520d9140}, 0x3c70) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc04e773500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc04e773500, {0xc0523f7000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0523ef080, {0xc01fd01e00, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0523ef080}, {0xc01fd01e00, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc01fd01e00, 0x9, 0xc00161e330}, {0x35449e0, 0xc0523ef080}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc01fd01dc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e610f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0523e6600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4589 [select]: | |
| net/http.(*persistConn).writeLoop(0xc032f24d80) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4567 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3758, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc048908300, 0xc04934a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc048908300, {0xc04934a000, 0x59f1, 0x59f1}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc048908300, {0xc04934a000, 0x59ec, 0xc049bc7c40}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01c177e88, {0xc04934a000, 0xc04934a000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc02d80fa58, {0xc04934a000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc043aa0cf8, {0x3544be0, 0xc02d80fa58}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc043aa0a80, {0x7f09e11af0a0, 0xc04806c3c0}, 0x59f1) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc043aa0a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc043aa0a80, {0xc049c3e000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc049c3c660, {0xc019aa2120, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc049c3c660}, {0xc019aa2120, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc019aa2120, 0x9, 0xc0017eb890}, {0x35449e0, 0xc049c3c660}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc019aa20e0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc02e503f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc049b7a480) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4399 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03e399200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852789 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732618, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc024d72d80, 0xc04212d000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc024d72d80, {0xc04212d000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc024d72d80, {0xc04212d000, 0xc032f59180, 0xc01d8c9f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013909a48, {0xc04212d000, 0xc0bb2b0340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc115b04900, {0xc04212d000, 0xc0621d7e60, 0xc01d8c9d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc02c5c6840) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc02c5c6840, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc115b04900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67109 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab450, 0xc0b2889590) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5040 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0316959e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10807231 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0600f9200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 92383 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05583f8c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10806750 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d25910, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06818fe80, 0xc0d2308000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06818fe80, {0xc0d2308000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06818fe80, {0xc0d2308000, 0xc032f53180, 0xc01e5d4f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0d18a1b08, {0xc0d2308000, 0xc017cdc1a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06698b0e0, {0xc0d2308000, 0xc092b89b60, 0xc01e5d4d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc066750960) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc066750960, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06698b0e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852303 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 4612 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0567659e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5892 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2bec0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a20c7000, 0xc0a11ee000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a20c7000, {0xc0a11ee000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a20c7000, {0xc0a11ee000, 0xc032f53a40, 0xc0a119af68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017b06d48, {0xc0a11ee000, 0xc014cc1ba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc051bb7440, {0xc0a11ee000, 0xc06b6d3680, 0xc0a119ad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a118c900) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a118c900, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc051bb7440) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67572 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d25828, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ccd21a80, 0xc0bf7ad000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ccd21a80, {0xc0bf7ad000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ccd21a80, {0xc0bf7ad000, 0xc032f58c80, 0xc020dc4f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04cbd1350, {0xc0bf7ad000, 0xc02a38dd40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02f0dd9e0, {0xc0bf7ad000, 0xc0bcbc70e0, 0xc020dc4d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0bf03d680) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0bf03d680, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02f0dd9e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4444 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02dfed560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67944 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd1ad0, 0xc0befa6e00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4895 [select]: | |
| net/http.(*persistConn).writeLoop(0xc027efa7e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10823068 [sync.Cond.Wait, 6 minutes]: | |
| sync.runtime_notifyListWait(0xc08957db48, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc08957db38) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08957db30, {0xc08d43b400, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc08d43b400, 0x217c472, 0xc007edecf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05f38e500) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05f38e500) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05f38e500, {0x2748340, 0xc03dc16168}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03e23af60, {0xc08bc4c800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0c9ab9540, 0xc007edee38, {0x357ac00, 0xc0530f8580}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0992dbf80) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0530f8540) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 5058 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fefb8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06a459880, 0xc05ee87000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06a459880, {0xc05ee87000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06a459880, {0xc05ee87000, 0x4c50, 0xc06f817360}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01906c4a8, {0xc05ee87000, 0xc05ee87000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020ca7590, {0xc05ee87000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc06f7925f8, {0x3544be0, 0xc020ca7590}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc06f792380, {0x7f09e11af0a0, 0xc06f87f680}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc06f792380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc06f792380, {0xc06f930000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc06f92e480, {0xc02afa7c40, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc06f92e480}, {0xc02afa7c40, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02afa7c40, 0x9, 0xc0017296e0}, {0x35449e0, 0xc06f92e480}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02afa7c00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01dd6df98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc06f2f0900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5724 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04f2edb00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5234 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d25740, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc079daeb80, 0xc076ab6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc079daeb80, {0xc076ab6000, 0x3e9e, 0x3e9e}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc079daeb80, {0xc076ab6000, 0x3e99, 0xc076a81180}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01e632730, {0xc076ab6000, 0xc076ab6000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc046f33ec0, {0xc076ab6000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc069db6cf8, {0x3544be0, 0xc046f33ec0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc069db6a80, {0x7f09e11af0a0, 0xc076a8a4c0}, 0x3e9e) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc069db6a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc069db6a80, {0xc076ab1000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc076aae660, {0xc02c194040, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc076aae660}, {0xc02c194040, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02c194040, 0x9, 0xc001a52300}, {0x35449e0, 0xc076aae660}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02c194000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e600f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc06a4da480) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 92832 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732530, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ac0c5480, 0xc0ac2a2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ac0c5480, {0xc0ac2a2000, 0xcad0, 0xcad0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ac0c5480, {0xc0ac2a2000, 0xc534, 0xc0ac194400}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0570b3850, {0xc0ac2a2000, 0xc0ac2a2597, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01c966018, {0xc0ac2a2000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0abd15078, {0x3544be0, 0xc01c966018}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0abd14e00, {0x7f09e11af0a0, 0xc0ac0428c0}, 0xc539) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0abd14e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0abd14e00, {0xc0ac1ca000, 0x1000, 0x472969}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0ac1b76e0, {0xc04399fee0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0ac1b76e0}, {0xc04399fee0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc04399fee0, 0x9, 0xc001b3ad20}, {0x35449e0, 0xc0ac1b76e0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc04399fea0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01a908f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0ac1b8180) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4048295 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc08a78f520, 0xc0046bc480) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5561 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2ad88, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09b417380, 0xc09ba60000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09b417380, {0xc09ba60000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09b417380, {0xc09ba60000, 0xc032f537c0, 0xc01bba2f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00f028288, {0xc09ba60000, 0xc014aa1520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04a1a5560, {0xc09ba60000, 0xc09307fda0, 0xc01bba2d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc09b92af00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc09b92af00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04a1a5560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 93770 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0296b8900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5072 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fec18, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06a459a80, 0xc072900000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06a459a80, {0xc072900000, 0x57a9, 0x57a9}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06a459a80, {0xc072900000, 0x246c, 0xc071394480}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01906ce30, {0xc072900000, 0xc072903338, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020be32a8, {0xc072900000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc06f793e78, {0x3544be0, 0xc020be32a8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc06f793c00, {0x7f09e11af0a0, 0xc071390980}, 0x2471) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc06f793c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc06f793c00, {0xc0713be000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0713bc120, {0xc02b1f03c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0713bc120}, {0xc02b1f03c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02b1f03c0, 0x9, 0xc0011345a0}, {0x35449e0, 0xc0713bc120}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02b1f0380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01c7bdf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc070a00900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4962 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18ff270, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06409bf80, 0xc0654f7000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06409bf80, {0xc0654f7000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06409bf80, {0xc0654f7000, 0xc033083a40, 0xc01a25ff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0176f90e8, {0xc0654f7000, 0xc0004556c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02bdf2480, {0xc0654f7000, 0xc06c6401e0, 0xc01a25fd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc05fd77200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc05fd77200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02bdf2480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4963 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02bdf2480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10790983 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08e43f320) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852206 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc01fa8b6c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01fa8b6b8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc01fa8b6b0, {0xc043992a00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc00ccc7cd8}, {0xc043992a00, 0x217c472, 0xc00ccc7cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0d85c4c80) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0d85c4c80) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0d85c4c80, {0x2748340, 0xc020bb5818}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0d6666b10, {0xc0602dc000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc020bc8640, 0xc00ccc7e38, {0x357ac00, 0xc020b2d680}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc020b2f6e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc020b2d640) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 5695 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c260, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a16daa00, 0xc05edac400, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a16daa00, {0xc05edac400, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a16daa00, {0xc05edac400, 0x7d7, 0xc0a10c46c0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0138fe5f0, {0xc05edac400, 0xc05edac43b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01d975998, {0xc05edac400, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc09bedf778, {0x3544be0, 0xc01d975998}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc09bedf500, {0x7f09e11af0a0, 0xc0a10a5540}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc09bedf500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc09bedf500, {0xc0a10eb000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a10df620, {0xc02dbf9c40, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a10df620}, {0xc02dbf9c40, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02dbf9c40, 0x9, 0xc001db9e90}, {0x35449e0, 0xc0a10df620}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02dbf9c00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003afbf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc09bf45c80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 71440 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b698, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc08735e600, 0xc087389000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc08735e600, {0xc087389000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc08735e600, {0xc087389000, 0xc032f59540, 0xc02045df68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc051526730, {0xc087389000, 0xc02a38dba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03bfe4a20, {0xc087389000, 0xc0d7f54600, 0xc02045dd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0e18467e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0e18467e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03bfe4a20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852790 [select]: | |
| net/http.(*persistConn).writeLoop(0xc115b04900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5935 [select]: | |
| net/http.(*persistConn).writeLoop(0xc00505ac60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10838242 [select, 4 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08289e780, 0xc0824f5500) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x28, 0xc08575b340) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 100732 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0500c91e0, 0xc06657a180) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 92853 [select]: | |
| net/http.(*persistConn).writeLoop(0xc039c1e5a0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6785 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730680, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02b696c80, 0xc0fc284000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02b696c80, {0xc0fc284000, 0x3cae, 0x3cae}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02b696c80, {0xc0fc284000, 0x3ca9, 0xc01f6bed60}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0156d4118, {0xc0fc284000, 0xc0fc284000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020f43db8, {0xc0fc284000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01fd3c5f8, {0x3544be0, 0xc020f43db8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01fd3c380, {0x7f09e11af0a0, 0xc05a5401c0}, 0x3cae) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01fd3c380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01fd3c380, {0xc0172b6000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc05a37fda0, {0xc02295d380, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc05a37fda0}, {0xc02295d380, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02295d380, 0x9, 0xc0013872c0}, {0x35449e0, 0xc05a37fda0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02295d340) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01ac76f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0148a3c80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5458 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24b78, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc096b8d600, 0xc097756000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc096b8d600, {0xc097756000, 0x4c4b, 0x4c4b}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc096b8d600, {0xc097756000, 0x4c46, 0xc09a80a820}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0179c01c8, {0xc097756000, 0xc097756000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc021fb9980, {0xc097756000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc09a9b6278, {0x3544be0, 0xc021fb9980}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc09a9b6000, {0x7f09e11af0a0, 0xc09a8067c0}, 0x4c4b) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc09a9b6000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc09a9b6000, {0xc09a9c5000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09a9bc0c0, {0xc02c9e1d20, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09a9bc0c0}, {0xc02c9e1d20, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02c9e1d20, 0x9, 0xc001fb8210}, {0x35449e0, 0xc09a9bc0c0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02c9e1ce0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003b86f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc096f28480) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6009 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644880, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc012bdb880, 0xc0ff7ec000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc012bdb880, {0xc0ff7ec000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc012bdb880, {0xc0ff7ec000, 0xc032f52f00, 0xc0185f9f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0180ce4b0, {0xc0ff7ec000, 0xc006f78d00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05f6827e0, {0xc0ff7ec000, 0xc092443680, 0xc0185f9d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a91c2de0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a91c2de0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05f6827e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67573 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02f0dd9e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10825406 [sync.Cond.Wait, 6 minutes]: | |
| sync.runtime_notifyListWait(0xc02983b9c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc02983b9b8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc02983b9b0, {0xc079d5d200, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc003b04cd8}, {0xc079d5d200, 0xc003b04d60, 0xc003b04cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc033dd4000) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc033dd4000) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc033dd4000, {0x2748340, 0xc02b192ae0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc094283ce0, {0xc040b19800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc02b1940f0, 0xc0135d19c0, {0x357ac00, 0xc02af97fc0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc02af718a0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02af97f80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 92727 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6328, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ab778180, 0xc0abd4e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ab778180, {0xc0abd4e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ab778180, {0xc0abd4e000, 0xc033083cc0, 0xc01a25df68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0570b3310, {0xc0abd4e000, 0xc0148f7380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc048f298c0, {0xc0abd4e000, 0xc01fc58f60, 0xc01a25dd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0abc1d2c0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0abc1d2c0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc048f298c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5521 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b2f8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09afa7100, 0xc09aebe000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09afa7100, {0xc09aebe000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09afa7100, {0xc09aebe000, 0xc033082b40, 0xc01cb52f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01d8c3ea0, {0xc09aebe000, 0xc02531cb60, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc049d0f8c0, {0xc09aebe000, 0xc028f5d7a0, 0xc01cb52d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc09aebc060) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc09aebc060, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc049d0f8c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67723 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05fd4b1e0, 0xc0c27d8870) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10843876 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc063ba67c8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc0014c3ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc063ba67b0, {0xc0b126e601, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc0b126e601, 0xc0a09f2600, 0xc0014c3cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc07bf823c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc07bf823c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc07bf823c0, {0x2748340, 0xc0257ede48}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc06ead4e40, {0xc0869d0800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc063d22460, 0xc01c298d80, {0x357ac00, 0xc0257a0140}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0636c8980) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc063d26ac0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 5518 [select]: | |
| net/http.(*persistConn).writeLoop(0xc049f74480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10835303 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc009e04048, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc02008bba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc009e04030, {0xc093655600, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc003aa24d8}, {0xc093655600, 0xc003aa2560, 0xc003aa24f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc05cc42140) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc05cc42140) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc05cc42140, {0x2748340, 0xc097885bc0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc093c8e090, {0xc03b8e1800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05ed57220, 0xc0135d1c80, {0x357ac00, 0xc0a6bb52c0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc06ce70660) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0a6bb5280) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 5566 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2abb8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09c17e580, 0xc0790d8000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09c17e580, {0xc0790d8000, 0x3c76, 0x3c76}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09c17e580, {0xc0790d8000, 0x3c71, 0xc09b997580}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00f028290, {0xc0790d8000, 0xc0790d8000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc097a303f0, {0xc0790d8000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc09777be78, {0x3544be0, 0xc097a303f0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc09777bc00, {0x7f09e11af0a0, 0xc09ba58a80}, 0x3c76) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc09777bc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc09777bc00, {0xc09ba69000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09ba62f60, {0xc02d97a900, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09ba62f60}, {0xc02d97a900, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02d97a900, 0x9, 0xc001eeb7a0}, {0x35449e0, 0xc09ba62f60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02d97a8c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e5d8f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc09ba31800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5517 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2af58, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09b32df80, 0xc09b440000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09b32df80, {0xc09b440000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09b32df80, {0xc09b440000, 0xc032f59b80, 0xc020839f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0179c08a0, {0xc09b440000, 0xc02c866ea0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc049f74480, {0xc09b440000, 0xc029d25a40, 0xc020839d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc09af91f20) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc09af91f20, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc049f74480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 95455 [select]: | |
| net/http.(*persistConn).writeLoop(0xc027e44000) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5562 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04a1a5560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6010 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05f6827e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6407 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2aad0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc014a01f80, 0xc0fb5f6900, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc014a01f80, {0xc0fb5f6900, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc014a01f80, {0xc0fb5f6900, 0x7d7, 0xc0162d90a0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc014ed1ac8, {0xc0fb5f6900, 0xc0fb5f693b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01fe30ac8, {0xc0fb5f6900, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01aa6b3f8, {0x3544be0, 0xc01fe30ac8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01aa6b180, {0x7f09e11af0a0, 0xc029e2b880}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01aa6b180, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01aa6b180, {0xc0cd811000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc014a7ede0, {0xc0327484a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc014a7ede0}, {0xc0327484a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0327484a0, 0x9, 0xc00164fe90}, {0x35449e0, 0xc014a7ede0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc032748460) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00c477f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0148a2d80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5543 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2aca0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09b4e1900, 0xc076293000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09b4e1900, {0xc076293000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09b4e1900, {0xc076293000, 0x4c50, 0xc09c000500}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0179c1ba0, {0xc076293000, 0xc076293000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0243840a8, {0xc076293000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc09bede5f8, {0x3544be0, 0xc0243840a8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc09bede380, {0x7f09e11af0a0, 0xc09bfc0a00}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc09bede380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc09bede380, {0xc09c012000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09c0034a0, {0xc02dc5b2a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09c0034a0}, {0xc02dc5b2a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02dc5b2a0, 0x9, 0xc0011e65d0}, {0x35449e0, 0xc09c0034a0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02dc5b260) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01cacdf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc09bf45080) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 93769 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bacf8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0dfa51680, 0xc0dfd54000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0dfa51680, {0xc0dfd54000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0dfa51680, {0xc0dfd54000, 0xc033083400, 0xc01e5dbf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc03c1c6048, {0xc0dfd54000, 0xc05e73da00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0296b8900, {0xc0dfd54000, 0xc0dfa825a0, 0xc01e5dbd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0dfc2a2a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0dfc2a2a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0296b8900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3966172 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc055e5c8f0, 0xc045730570) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 5881 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2bb20, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a2ad9d00, 0xc0654e0000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a2ad9d00, {0xc0654e0000, 0x3d0a, 0x3d0a}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a2ad9d00, {0xc0654e0000, 0x3d05, 0xc0a2e64720}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00f029b28, {0xc0654e0000, 0xc0654e0000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01df18d38, {0xc0654e0000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a2c18cf8, {0x3544be0, 0xc01df18d38}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a2c18a80, {0x7f09e11af0a0, 0xc0a2e531c0}, 0x3d0a) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a2c18a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a2c18a80, {0xc0a2e7a000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a2e78b40, {0xc02ed169e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a2e78b40}, {0xc02ed169e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02ed169e0, 0x9, 0xc0012e1830}, {0x35449e0, 0xc0a2e78b40}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02ed169a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003a70f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a2e00f00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5725 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c348, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a12d0180, 0xc0a1188000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a12d0180, {0xc0a1188000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a12d0180, {0xc0a1188000, 0xc032f59b80, 0xc04aa1ff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017b06c70, {0xc0a1188000, 0xc013ff4000, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc04eee98c0, {0xc0a1188000, 0xc031af73e0, 0xc04aa1fd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a0a41e00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a0a41e00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc04eee98c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 92259 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15a28, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0891a1180, 0xc0923de000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0891a1180, {0xc0923de000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0891a1180, {0xc0923de000, 0xc032f59540, 0xc02e505f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc053c58da8, {0xc0923de000, 0xc036a1e4e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06328e240, {0xc0923de000, 0xc0ecf30120, 0xc02e505d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0923d4240) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0923d4240, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06328e240) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852747 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15eb0, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc025094200, 0xc05cd74990, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc025094200, {0x35a0d00, 0xc02745bc20}, {0xc0171a52f8, 0x40ce34}, {0x354bea0, 0xc0bc1e33a0}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc025094200, {0x35a0d00, 0xc02745bc20}, {0x35c3098, 0x0}, {0x35c3098, 0xc05cd74960}, 0xc0171a54e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc05cd74960}, 0xc0018e9b10, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc025094180, {0x35a0d00, 0xc02745bc20}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc02745bc20, {0x35a0d00, 0xc02745bc20}, 0xc01d8c95a8, 0x0) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc025094180, {0x35a0d00, 0xc02745bc20}, {0x357b3f8, 0xc05cd74960}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc025094180, {0x35a0d00, 0xc02745bc20}, {0xc026cfb830, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc019784a80, {0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0xc026cfb810}, {0xc0235446d8, 0xc016f509a0}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0x3}, {0xc0235446d8, 0x13}, {0xc017c69860, 0x0}, 0xc019563e40) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0x3}, {0xc0235446d8, 0x13}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc02745bc20, {0x35a0d00, 0xc02745bc20}, {0x2c655e4, 0xc0a345a000}, {0xc0235446d8, 0x203009}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f52280, {0x35a0d00, 0xc02745bc20}, {{}, 0x0, {0xc041ea24c0, 0x5}, {0xc0235446d8, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x35a16a0, 0xc0ecafc580) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 5845 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2bfa8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a20c6f00, 0xc0a22c5000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a20c6f00, {0xc0a22c5000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a20c6f00, {0xc0a22c5000, 0xc032f58780, 0xc0a119ef68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0138ff448, {0xc0a22c5000, 0xc02efd6000, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc051bb7320, {0xc0a22c5000, 0xc031af79e0, 0xc0a119ed30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a2343500) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a2343500, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc051bb7320) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5852 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2ba38, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a37eca80, 0xc05edacd00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a37eca80, {0xc05edacd00, 0x815, 0x815}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a37eca80, {0xc05edacd00, 0x810, 0xc0a2bd5480}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0138ff6f8, {0xc05edacd00, 0xc05edacd00, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01d9ff368, {0xc05edacd00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a2978978, {0x3544be0, 0xc01d9ff368}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a2978700, {0x7f09e11af0a0, 0xc0a2b03ec0}, 0x815) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a2978700, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a2978700, {0xc0a3909000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a3904900, {0xc02ecd8660, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a3904900}, {0xc02ecd8660, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02ecd8660, 0x9, 0xc00198d620}, {0x35449e0, 0xc0a3904900}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02ecd8620) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0713c3f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a1a6ac00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 94767 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04f1938c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 5887 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0505d1e60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 69690 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d253a0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ca74e180, 0xc0b1f19000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ca74e180, {0xc0b1f19000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ca74e180, {0xc0b1f19000, 0xc032f58dc0, 0xc02045bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04ac667f0, {0xc0b1f19000, 0xc013c09ba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0276f2240, {0xc0b1f19000, 0xc090d44360, 0xc02045bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc024c93d40) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc024c93d40, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0276f2240) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5961 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0645278, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a5d4d900, 0xc0a6a8c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a5d4d900, {0xc0a6a8c000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a5d4d900, {0xc0a6a8c000, 0x3ccb, 0xc0a50e1da0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013faf160, {0xc0a6a8c000, 0xc0a6a8c000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01e03d8a8, {0xc0a6a8c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a5282cf8, {0x3544be0, 0xc01e03d8a8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a5282a80, {0x7f09e11af0a0, 0xc0a6a523c0}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a5282a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a5282a80, {0xc0a5e40000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a5e14720, {0xc0306137e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a5e14720}, {0xc0306137e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0306137e0, 0x9, 0xc0012e9ce0}, {0x35449e0, 0xc0a5e14720}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0306137a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0a6065f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a519e180) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 5947 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644fc0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a7396c80, 0xc0a7758000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a7396c80, {0xc0a7758000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a7396c80, {0xc0a7758000, 0x3ccb, 0xc0a6a59a20}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01499e6c8, {0xc0a7758000, 0xc0a7758000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01ece99c8, {0xc0a7758000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc09ec51e78, {0x3544be0, 0xc01ece99c8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc09ec51c00, {0x7f09e11af0a0, 0xc0a4e57540}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc09ec51c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc09ec51c00, {0xc0a7753000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a7750360, {0xc030993620, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a7750360}, {0xc030993620, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc030993620, 0x9, 0xc001c69a40}, {0x35449e0, 0xc0a7750360}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0309935e0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e607f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a7390300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 67233 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2bc08, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0befa6480, 0xc0bf106000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0befa6480, {0xc0bf106000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0befa6480, {0xc0bf106000, 0xc033082c80, 0xc01e605f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04cbd01e0, {0xc0bf106000, 0xc054f02680, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06ef14900, {0xc0bf106000, 0xc0c0e8b020, 0xc01e605d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0bf03c600) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0bf03c600, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06ef14900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6053 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644968, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0196cb100, 0xc0a2562000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0196cb100, {0xc0a2562000, 0x813, 0x813}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0196cb100, {0xc0a2562000, 0x80e, 0xc0a9265500}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0182b0d60, {0xc0a2562000, 0xc0a2562000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01e1152d8, {0xc0a2562000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a90d8978, {0x3544be0, 0xc01e1152d8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a90d8700, {0x7f09e11af0a0, 0xc0f247e500}, 0x813) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a90d8700, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a90d8700, {0xc0fda4c000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc000090240, {0xc03163eac0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc000090240}, {0xc03163eac0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc03163eac0, 0x9, 0xc0019b04b0}, {0x35449e0, 0xc000090240}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc03163ea80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc02080cf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0a1a6b680) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 113975 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734350, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0af878980, 0xc0c46c2000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0af878980, {0xc0c46c2000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0af878980, {0xc0c46c2000, 0xc032f53b80, 0xc003a35f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc05a8860e0, {0xc0c46c2000, 0xc014851380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0411a0ea0, {0xc0c46c2000, 0xc0242dc420, 0xc003a35d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc07f5bc540) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc07f5bc540, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0411a0ea0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10848421 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc048797680, 0xc048979300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc0c5d4efc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 6432 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e07328d0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc018d36700, 0xc0fe821b00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc018d36700, {0xc0fe821b00, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc018d36700, {0xc0fe821b00, 0x7d3, 0xc017b356a0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013aca2d0, {0xc0fe821b00, 0xc0fe821b3f, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01fd58dc8, {0xc0fe821b00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01e3fbaf8, {0x3544be0, 0xc01fd58dc8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01e3fb880, {0x7f09e11af0a0, 0xc02bcbe580}, 0x7d8) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01e3fb880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01e3fb880, {0xc0d4b4d000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0218ac120, {0xc02f1f9620, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0218ac120}, {0xc02f1f9620, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02f1f9620, 0x9, 0xc0017c32c0}, {0x35449e0, 0xc0218ac120}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02f1f95e0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e614f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc017a83200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 66947 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc10cae2410, 0xc021054f00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 145974 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01c783450, 0x4dbf460) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10807230 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2a818, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc085777180, 0xc07c4e7000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc085777180, {0xc07c4e7000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc085777180, {0xc07c4e7000, 0xc032f53180, 0xc003b77f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0a18fc640, {0xc07c4e7000, 0xc036d8d040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0600f9200, {0xc07c4e7000, 0xc084ebd260, 0xc003b77d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc07c467aa0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc07c467aa0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0600f9200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 9721093 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b5730, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc027ab3a00, 0xc050380000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc027ab3a00, {0xc050380000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc027ab3a00, {0xc050380000, 0x3ccb, 0xc04f88b1e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0effbf438, {0xc050380000, 0xc050380000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020c6ed20, {0xc050380000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc050312278, {0x3544be0, 0xc020c6ed20}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc050312000, {0x7f09e11af0a0, 0xc0503101c0}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc050312000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc050312000, {0xc010653000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc050315f80, {0xc0444b30e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc050315f80}, {0xc0444b30e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0444b30e0, 0x9, 0xc001591800}, {0x35449e0, 0xc050315f80}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0444b30a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01a90cf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc045347800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6115 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb520, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02907fa80, 0xc0afa66000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02907fa80, {0xc0afa66000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02907fa80, {0xc0afa66000, 0x3ccb, 0xc01162b4e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc01499edf0, {0xc0afa66000, 0xc0afa66000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01f0a08d0, {0xc0afa66000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc003a6de78, {0x3544be0, 0xc01f0a08d0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc003a6dc00, {0x7f09e11af0a0, 0xc017633700}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc003a6dc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc003a6dc00, {0xc01073a000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc003d9f260, {0xc016933b60, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc003d9f260}, {0xc016933b60, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc016933b60, 0x9, 0xc001857590}, {0x35449e0, 0xc003d9f260}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc016933b20) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0037cff98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02af17200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6129 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb438, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05dea1c00, 0xc068f31000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05dea1c00, {0xc068f31000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05dea1c00, {0xc068f31000, 0xc032f592c0, 0xc01ab85f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0180cfe28, {0xc068f31000, 0xc02e625d40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0023e1d40, {0xc068f31000, 0xc0202904e0, 0xc01ab85d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc003c142a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc003c142a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0023e1d40) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6130 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0023e1d40) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6107 [select]: | |
| net/http.(*persistConn).writeLoop(0xc002499560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852302 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0731418, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc007289680, 0xc016690420, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc007289680, {0x35a0d00, 0xc020bdf0e0}, {0xc00e51f2f8, 0x40ce34}, {0x354bea0, 0xc097987c80}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc007289680, {0x35a0d00, 0xc020bdf0e0}, {0x35c3098, 0x0}, {0x35c3098, 0xc0166903f0}, 0xc00e51f4e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0166903f0}, 0xc01530f8d0, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc007289600, {0x35a0d00, 0xc020bdf0e0}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc020bdf0e0, {0x35a0d00, 0xc020bdf0e0}, 0x40ce34, 0x1) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc007289600, {0x35a0d00, 0xc020bdf0e0}, {0x357b3f8, 0xc0166903f0}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc007289600, {0x35a0d00, 0xc020bdf0e0}, {0xc020ea1fe0, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc01978eea0, {0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0xc020ea1fc0}, {0xc00fab7d88, 0xc001c95ce0}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0x3}, {0xc00fab7d88, 0x13}, {0xc017c69590, 0x0}, 0xc01978aec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0x3}, {0xc00fab7d88, 0x13}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc020bdf0e0, {0x35a0d00, 0xc020bdf0e0}, {0x2c655e4, 0x0}, {0xc00fab7d88, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f53900, {0x35a0d00, 0xc020bdf0e0}, {{}, 0x0, {0xc0189b8840, 0x5}, {0xc00fab7d88, 0x13}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x6ce866, 0xc0252aab00) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 91267 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a20969c0, 0xc08ea575c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10843400 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc05d5c81c8, 0xa) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc05d5c81b0, {0xc003d9a001, 0x1dff, 0x1dff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc003d9a001, 0xc003d9a000, 0xc01449dcf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0711b6000) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0711b6000) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0711b6000, {0x2748340, 0xc020f18b88}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc097f55620, {0xc031b4c600, 0x2000, 0x2600}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05cea3f90, 0xc045df17a0, {0x357ac00, 0xc020ee3c40}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05ce593e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc05d5a0940) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 6160 [select]: | |
| net/http.(*persistConn).writeLoop(0xc00c613200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 91220 [select]: | |
| net/http.(*persistConn).writeLoop(0xc04a2b2fc0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6111 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb180, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05e5c7b80, 0xc0f4910000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05e5c7b80, {0xc0f4910000, 0x3ccc, 0x3ccc}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05e5c7b80, {0xc0f4910000, 0x3cc7, 0xc0038c1480}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f275a8, {0xc0f4910000, 0xc0f4910000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01f582138, {0xc0f4910000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0a90d9e78, {0x3544be0, 0xc01f582138}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0a90d9c00, {0x7f09e11af0a0, 0xc0219ce2c0}, 0x3ccc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0a90d9c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0a90d9c00, {0xc014113000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc003c25c20, {0xc01e136660, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc003c25c20}, {0xc01e136660, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc01e136660, 0x9, 0xc00151fce0}, {0x35449e0, 0xc003c25c20}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc01e136620) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc003adcf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc04ecde600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6106 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb268, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc05e5c7200, 0xc052ef9000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc05e5c7200, {0xc052ef9000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc05e5c7200, {0xc052ef9000, 0xc032f58780, 0xc0209b2f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f275a0, {0xc052ef9000, 0xc02efd6820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc002499560, {0xc052ef9000, 0xc014db9c20, 0xc0209b2d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc003a79b00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc003a79b00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc002499560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 94820 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f9f0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc023446a00, 0xc09aec1000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc023446a00, {0xc09aec1000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc023446a00, {0xc09aec1000, 0xc032f537c0, 0xc01e5ccf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc050f7fba8, {0xc09aec1000, 0xc036d8c340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc080292480, {0xc09aec1000, 0xc067d3e420, 0xc01e5ccd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0219dcc00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0219dcc00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc080292480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10828887 [select, 6 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08db32480, 0xc0860fed00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0xc083f27c20, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 10852904 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18ff0a0, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc060702080, 0xc0f563fbc0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc060702080, {0x35a0d00, 0xc03973b9e0}, {0xc087a012f8, 0x40ce34}, {0x354bea0, 0xc07b0a5680}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc060702080, {0x35a0d00, 0xc03973b9e0}, {0x35c3098, 0x0}, {0x35c3098, 0xc0f563fb90}, 0xc087a014e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc0f563fb90}, 0xc089879930, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc060702000, {0x35a0d00, 0xc03973b9e0}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc03973b9e0, {0x35a0d00, 0xc03973b9e0}, 0x416f26, 0x2000) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc060702000, {0x35a0d00, 0xc03973b9e0}, {0x357b3f8, 0xc0f563fb90}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc060702000, {0x35a0d00, 0xc03973b9e0}, {0xc0397473a0, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc01978eea0, {0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0xc039747380}, {0xc020225c98, 0xc09afe5080}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0x3}, {0xc020225c98, 0x12}, {0xc017c69590, 0x0}, 0xc01978aec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0x3}, {0xc020225c98, 0x12}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc03973b9e0, {0x35a0d00, 0xc03973b9e0}, {0x2c655e4, 0x0}, {0xc020225c98, 0x7f09e62a69b0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f53900, {0x35a0d00, 0xc03973b9e0}, {{}, 0x0, {0xc043b35380, 0x5}, {0xc020225c98, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x82b7a6, 0xc023ff0bb0) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 5934 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05baec8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a04a6600, 0xc00df08000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a04a6600, {0xc00df08000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a04a6600, {0xc00df08000, 0xc032f58780, 0xc0209b1f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0138fe518, {0xc00df08000, 0xc02efd64e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc00505ac60, {0xc00df08000, 0xc021f47320, 0xc0209b1d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a395e480) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a395e480, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc00505ac60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 69691 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0276f2240) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 94821 [select]: | |
| net/http.(*persistConn).writeLoop(0xc080292480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10816624 [select, 9 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc06e1b6a80, 0xc07189c000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc071648b40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 6245 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0057d6b40) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6244 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05baa40, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00a9b2c80, 0xc04f203000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00a9b2c80, {0xc04f203000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00a9b2c80, {0xc04f203000, 0xc032f58780, 0xc0209bcf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f276a8, {0xc04f203000, 0xc02f01ba00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0057d6b40, {0xc04f203000, 0xc024c06060, 0xc0209bcd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc007036240) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc007036240, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0057d6b40) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6256 [select]: | |
| net/http.(*persistConn).writeLoop(0xc005c40a20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 139897 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a5f88, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0165abc00, 0xc0a5e41000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0165abc00, {0xc0a5e41000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0165abc00, {0xc0a5e41000, 0xc032f59b80, 0xc01dd6af68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0677e1380, {0xc0a5e41000, 0xc02b05ba00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02c12d200, {0xc0a5e41000, 0xc01a1c60c0, 0xc01dd6ad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a84ab200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a84ab200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02c12d200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6255 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0731248, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c223400, 0xc099090000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c223400, {0xc099090000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c223400, {0xc099090000, 0xc032f53540, 0xc02008af68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f276e0, {0xc099090000, 0xc02b5feea0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc005c40a20, {0xc099090000, 0xc028350d20, 0xc02008ad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc009aed9e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc009aed9e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc005c40a20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6262 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730b08, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00f743100, 0xc0cacff000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00f743100, {0xc0cacff000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00f743100, {0xc0cacff000, 0xc032f58780, 0xc0a6265f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f277c8, {0xc0cacff000, 0xc02efd61a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc010832360, {0xc0cacff000, 0xc022e17ce0, 0xc0a6265d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc00c2f64e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc00c2f64e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc010832360) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6263 [select]: | |
| net/http.(*persistConn).writeLoop(0xc010832360) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 93231 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc071a24680, 0xc0170d0d80) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6159 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0731078, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00e59d680, 0xc0ae195000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00e59d680, {0xc0ae195000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00e59d680, {0xc0ae195000, 0xc032f59b80, 0xc02083bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0135adbc8, {0xc0ae195000, 0xc01b34a340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc00c613200, {0xc0ae195000, 0xc028765980, 0xc02083bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc00c5621e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc00c5621e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc00c613200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 135272 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0403d5320) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 92395 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0154bfb00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6267 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730a20, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00f203880, 0xc0c3364000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00f203880, {0xc0c3364000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00f203880, {0xc0c3364000, 0x3c6f, 0xc072c609e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0f6f277d0, {0xc0c3364000, 0xc0c3364000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01f7e5b90, {0xc0c3364000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc011ba9078, {0x3544be0, 0xc01f7e5b90}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc011ba8e00, {0x7f09e11af0a0, 0xc0250b02c0}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc011ba8e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc011ba8e00, {0xc0cc0ba000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc01319ec60, {0xc02c9c3e00, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc01319ec60}, {0xc02c9c3e00, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02c9c3e00, 0x9, 0xc0012791a0}, {0x35449e0, 0xc01319ec60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02c9c3dc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00c476f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc009276600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10852713 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 3959099 [select]: | |
| net/http.(*persistConn).writeLoop(0xc040d7d200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67002 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc08f73ef70, 0xc00e8487e0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6360 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0315cc6c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10600377 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09d74e1a0, 0xc0248805c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 7089 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0632b8120) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 91444 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a0ffe9c0, 0xc017dd5a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10847530 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc03f82c4c8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc000855ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc03f82c4b0, {0xc03c7d5801, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc03c7d5801, 0xc0799e8200, 0xc000855cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc066c9c140) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc066c9c140) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc066c9c140, {0x2748340, 0xc01e115338}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0284af590, {0xc00f846400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc040a9f540, 0xc054db22d0, {0x357ac00, 0xc01e1243c0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0408d1e20) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc040ac2bc0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 6720 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732c70, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02cd96e00, 0xc0fe822d00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02cd96e00, {0xc0fe822d00, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02cd96e00, {0xc0fe822d00, 0x7d7, 0xc027886380}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015b38c00, {0xc0fe822d00, 0xc0fe822d3b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc021fb9650, {0xc0fe822d00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0202ff078, {0x3544be0, 0xc021fb9650}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0202fee00, {0x7f09e11af0a0, 0xc059adaf40}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0202fee00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0202fee00, {0xc017590000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc063e37e00, {0xc009348e40, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc063e37e00}, {0xc009348e40, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc009348e40, 0x9, 0xc0016c3c50}, {0x35449e0, 0xc063e37e00}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc009348e00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01dd6bf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc020243380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10847450 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc04094a000, 0xc094822600) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0ce787a40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 6423 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732b88, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc017af9e00, 0xc028084000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc017af9e00, {0xc028084000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc017af9e00, {0xc028084000, 0x3c6f, 0xc017b353e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013aca2a0, {0xc028084000, 0xc028084000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0206b3050, {0xc028084000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01e3fb3f8, {0x3544be0, 0xc0206b3050}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01e3fb180, {0x7f09e11af0a0, 0xc02bcbe200}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01e3fb180, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01e3fb180, {0xc0cf356000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc017d45c20, {0xc02f1f9460, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc017d45c20}, {0xc02f1f9460, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc02f1f9460, 0x9, 0xc001d9d650}, {0x35449e0, 0xc017d45c20}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc02f1f9420) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e456f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc017a82d80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 7058 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06446b0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc030e09080, 0xc040a58000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc030e09080, {0xc040a58000, 0x3ccc, 0x3ccc}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc030e09080, {0xc040a58000, 0x3cc7, 0xc02eb2dd60}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0156d45a8, {0xc040a58000, 0xc040a58000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0229e8768, {0xc040a58000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01fd3de78, {0x3544be0, 0xc0229e8768}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01fd3dc00, {0x7f09e11af0a0, 0xc06b15dd80}, 0x3ccc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01fd3dc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01fd3dc00, {0xc05e540000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09bb484e0, {0xc01604c3c0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09bb484e0}, {0xc01604c3c0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc01604c3c0, 0x9, 0xc001024f90}, {0x35449e0, 0xc09bb484e0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc01604c380) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01b510f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02caed200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 7024 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734608, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc035064000, 0xc032428000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc035064000, {0xc032428000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc035064000, {0xc032428000, 0xc032f58780, 0xc0209baf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc016444330, {0xc032428000, 0xc02efd7380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0632b8120, {0xc032428000, 0xc098a757a0, 0xc0209bad30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a2a202a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a2a202a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0632b8120) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6534 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0158da480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 67110 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab520, 0xc06b5651a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6545 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732360, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0163afc80, 0xc0deb93000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0163afc80, {0xc0deb93000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0163afc80, {0xc0deb93000, 0xc032f59680, 0xc019891f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0152da280, {0xc0deb93000, 0xc01435eb60, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc00f4c30e0, {0xc0deb93000, 0xc02465db00, 0xc019891d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0152fd7a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0152fd7a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc00f4c30e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6533 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732278, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0198fd600, 0xc0efeac000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0198fd600, {0xc0efeac000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0198fd600, {0xc0efeac000, 0xc033082b40, 0xc02008df68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0157c9ac8, {0xc0efeac000, 0xc0147824e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0158da480, {0xc0efeac000, 0xc02d734ea0, 0xc02008dd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc023570c60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc023570c60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0158da480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6546 [select]: | |
| net/http.(*persistConn).writeLoop(0xc00f4c30e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 3960023 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b780, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0689e8a80, 0xc06f0fe000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0689e8a80, {0xc06f0fe000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0689e8a80, {0xc06f0fe000, 0x183f, 0xc06a875960}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0302dc5d0, {0xc06f0fe000, 0xc06f101411, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020c221f8, {0xc06f0fe000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc04ed425f8, {0x3544be0, 0xc020c221f8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc04ed42380, {0x7f09e11af0a0, 0xc06ab96c40}, 0x1844) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc04ed42380, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc04ed42380, {0xc0e12f6000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc06abb3d40, {0xc06a7229e0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc06abb3d40}, {0xc06a7229e0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc06a7229e0, 0x9, 0xc0015fa840}, {0x35449e0, 0xc06abb3d40}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc06a7229a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc020086f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc061aa2900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10847615 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0411ce648, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0411ce630, {0xc0fd076c01, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc0fd076c01, 0xc0a09f2000, 0xc01e11fcf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0958b77c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0958b77c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0958b77c0, {0x2748340, 0xc01fd58c18}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0cf945170, {0xc08b1c0c00, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0414bd360, 0xc067ea0d80, {0x357ac00, 0xc01fd5e780}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc041481440) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc041498f80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 6771 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732190, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc033713a80, 0xc070879000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc033713a80, {0xc070879000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc033713a80, {0xc070879000, 0xc032f52c80, 0xc0183eaf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015e94070, {0xc070879000, 0xc01376c4e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0628990e0, {0xc070879000, 0xc09ad986c0, 0xc0183ead30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc05e0194a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc05e0194a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0628990e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 5262879 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03a023e60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6538 [select]: | |
| net/http.(*persistConn).writeLoop(0xc015abe000) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6537 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e07320a8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc019dfb600, 0xc0f5596000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc019dfb600, {0xc0f5596000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc019dfb600, {0xc0f5596000, 0xc032f59b80, 0xc020819f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0157c9ad8, {0xc0f5596000, 0xc02c8664e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc015abe000, {0xc0f5596000, 0xc01d1dfb00, 0xc020819d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc023570de0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc023570de0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc015abe000) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 7022 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e07346f0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc027863180, 0xc09b7c0000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc027863180, {0xc09b7c0000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc027863180, {0xc09b7c0000, 0x3ccb, 0xc032406c20}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc016444310, {0xc09b7c0000, 0xc09b7c0000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0229f4cc0, {0xc09b7c0000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc02d38be78, {0x3544be0, 0xc0229f4cc0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc02d38bc00, {0x7f09e11af0a0, 0xc077b7ed00}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc02d38bc00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc02d38bc00, {0xc0a3065000, 0x1000, 0x472969}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a2a201e0, {0xc01afa1000, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a2a201e0}, {0xc01afa1000, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc01afa1000, 0x9, 0xc0011d2150}, {0x35449e0, 0xc0a2a201e0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc01afa0fc0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc02081af98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0320d7200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 4778404 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24a90, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0926a3680, 0xc04e7f0000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0926a3680, {0xc04e7f0000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0926a3680, {0xc04e7f0000, 0xc032f532c0, 0xc020e0cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04496c138, {0xc04e7f0000, 0xc02b8661a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0050c18c0, {0xc04e7f0000, 0xc091cb5c80, 0xc020e0cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0875da780) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0875da780, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0050c18c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 92728 [select]: | |
| net/http.(*persistConn).writeLoop(0xc048f298c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 92260 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06328e240) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6637 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0154e8480) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10838119 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc08289e7c8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xac) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc08289e7b0, {0xc09942cc01, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc09942cc01, 0xc00bb7f400, 0xc0014c5cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc066b5d2c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc066b5d2c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc066b5d2c0, {0x2748340, 0xc046f329a8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc073310de0, {0xc0965a7400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0728e5540, 0xc0e93d3b00, {0x357ac00, 0xc01adf49c0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc072ec4c40) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc072ec1d00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 53831 [select]: | |
| net/http.(*persistConn).writeLoop(0xc05f4f78c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6776 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e07347d8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc033b0a080, 0xc0fde10400, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc033b0a080, {0xc0fde10400, 0x819, 0x819}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc033b0a080, {0xc0fde10400, 0x7d5, 0xc02787e9c0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015e94078, {0xc0fde10400, 0xc0fde1043f, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0243e5cb0, {0xc0fde10400, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc024eaecf8, {0x3544be0, 0xc0243e5cb0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc024eaea80, {0x7f09e11af0a0, 0xc05a2a2780}, 0x7da) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc024eaea80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc024eaea80, {0xc09bf2a000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0a2343aa0, {0xc011239380, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0a2343aa0}, {0xc011239380, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc011239380, 0x9, 0xc001e65260}, {0x35449e0, 0xc0a2343aa0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc011239340) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e45af98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0233e9680) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6556 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0299084e0, 0x0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6678 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06057b0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02057fe00, 0xc01677e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02057fe00, {0xc01677e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02057fe00, {0xc01677e000, 0xc032f58780, 0xc020857f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015338ef8, {0xc01677e000, 0xc02efd76c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc01bc16c60, {0xc01677e000, 0xc03ebc0ba0, 0xc020857d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc02684e480) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc02684e480, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc01bc16c60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10606403 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc052d82820, 0x4dbf460) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10847614 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc0411ce600, 0xc06ba13f00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc0cf257b30) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 69622 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730f90, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc083d4a600, 0xc0cc5f8000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc083d4a600, {0xc0cc5f8000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc083d4a600, {0xc0cc5f8000, 0xc032f53a40, 0xc020e11f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc049d28ef0, {0xc0cc5f8000, 0xc02c640820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05cf75200, {0xc0cc5f8000, 0xc0b3cd35c0, 0xc020e11d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0213506c0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0213506c0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05cf75200) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6522 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc024d965b0, 0xc051bb7320) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10841059 [sync.Cond.Wait, 3 minutes]: | |
| sync.runtime_notifyListWait(0xc01120b3c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01ab84ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc01120b3b0, {0xc045afd800, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x495529}, {0xc045afd800, 0xc003aa3560, 0xc003aa34f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0932e23c0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0932e23c0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0932e23c0, {0x2748340, 0xc08973a108}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0813cb8c0, {0xc09f679000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0113e4be0, 0xc0135d1940, {0x357ac00, 0xc0113ebd00}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc066426300) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0113ebcc0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 6988 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732448, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc02ce2ff80, 0xc0b296c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc02ce2ff80, {0xc0b296c000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc02ce2ff80, {0xc0b296c000, 0x3c6f, 0xc02fca9400}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015b390d0, {0xc0b296c000, 0xc0b296c000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc022f0e4f8, {0xc0b296c000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc02fc9f778, {0x3544be0, 0xc022f0e4f8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc02fc9f500, {0x7f09e11af0a0, 0xc0718b59c0}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc02fc9f500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc02fc9f500, {0xc05a13c000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc09a99e3c0, {0xc0166332a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc09a99e3c0}, {0xc0166332a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc0166332a0, 0x9, 0xc001cc8fc0}, {0x35449e0, 0xc09a99e3c0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc016633260) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e5e3f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc020243800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 6772 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0628990e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10836297 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc02f995e48, 0x13) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0x16d9) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc02f995e30, {0xc0316d6001, 0x1dff, 0x1dff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0x16da}, {0xc0316d6001, 0xc0316d6000, 0xc050195cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0a7e8e000) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0a7e8e000) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0a7e8e000, {0x2748340, 0xc097cd33c8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc05fdab470, {0xc031b4a000, 0x2000, 0x2600}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc02fb5b400, 0xc097cf9800, {0x357ac00, 0xc097cc11c0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc02fb49220) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02fb47d40) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 10607130 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730dc0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc070e2ee80, 0xc0b7cd4000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc070e2ee80, {0xc0b7cd4000, 0x3cd0, 0x3cd0}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc070e2ee80, {0xc0b7cd4000, 0x3ccb, 0xc0f9e022e0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04107b460, {0xc0b7cd4000, 0xc0b7cd4000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc0a4e09830, {0xc0b7cd4000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc0250b5078, {0x3544be0, 0xc0a4e09830}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc0250b4e00, {0x7f09e11af0a0, 0xc0039cd7c0}, 0x3cd0) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc0250b4e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc0250b4e00, {0xc0ce270000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0eda5ecc0, {0xc047b28120, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0eda5ecc0}, {0xc047b28120, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc047b28120, 0x9, 0xc00195ad20}, {0x35449e0, 0xc0eda5ecc0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc047b280e0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0a6063f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0072fe180) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 90852 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02939d520, 0xc03bcc7ec0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6636 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605a68, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc019a31000, 0xc014de6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc019a31000, {0xc014de6000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc019a31000, {0xc014de6000, 0xc032f59b80, 0xc02084cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0152dbab8, {0xc014de6000, 0xc0286069c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0154e8480, {0xc014de6000, 0xc03919d800, 0xc02084cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc024ae8600) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc024ae8600, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0154e8480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6639 [select]: | |
| net/http.(*persistConn).writeLoop(0xc018b75c20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6638 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605980, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc020274600, 0xc01543a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc020274600, {0xc01543a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc020274600, {0xc01543a000, 0xc032f52c80, 0xc01988df68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0152dbac0, {0xc01543a000, 0xc01448f380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc018b75c20, {0xc01543a000, 0xc03cab4960, 0xc01988dd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc024ae86c0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc024ae86c0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc018b75c20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 69614 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0682eab40) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6679 [select]: | |
| net/http.(*persistConn).writeLoop(0xc01bc16c60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 6759 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605158, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc020275280, 0xc0171c4000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc020275280, {0xc0171c4000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc020275280, {0xc0171c4000, 0xc032f58780, 0xc02045cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015d9f318, {0xc0171c4000, 0xc0f46371e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03ebf99e0, {0xc0171c4000, 0xc06fc0a180, 0xc02045cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc052f42600) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc052f42600, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03ebf99e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 90866 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc048e6c410, 0xc020359a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6690 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bade0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc020946200, 0xc0fb80c400, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc020946200, {0xc0fb80c400, 0x813, 0x813}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc020946200, {0xc0fb80c400, 0x7d3, 0xc020561640}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015d9ef20, {0xc0fb80c400, 0xc0fb80c43b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020d66a68, {0xc0fb80c400, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc020153778, {0x3544be0, 0xc020d66a68}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc020153500, {0x7f09e11af0a0, 0xc04806c1c0}, 0x7d8) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc020153500, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc020153500, {0xc016a24000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc029c8a8a0, {0xc009dfa900, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc029c8a8a0}, {0xc009dfa900, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc009dfa900, 0x9, 0xc001082d20}, {0x35449e0, 0xc029c8a8a0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc009dfa8c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01a136f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02017fb00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10835302 [select, 4 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc009e04000, 0xc09cd4e000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc0437226c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 67185 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18fe790, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b325fb80, 0xc0be88e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b325fb80, {0xc0be88e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b325fb80, {0xc0be88e000, 0xc033083400, 0xc01a25ef68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04ccc6038, {0xc0be88e000, 0xc02b5ff860, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc06c9f6360, {0xc0be88e000, 0xc0b8007620, 0xc01a25ed30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0be88c1e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0be88c1e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc06c9f6360) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 7093 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0632fde60) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 439796 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2c518, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc019f85080, 0xc09b368000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc019f85080, {0xc09b368000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc019f85080, {0xc09b368000, 0xc032f597c0, 0xc0a119cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc02a5a2cb0, {0xc09b368000, 0xc02f01bba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc007f1ad80, {0xc09b368000, 0xc02513ca20, 0xc0a119cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc01285d920) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc01285d920, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc007f1ad80) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 7092 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734438, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc03518a500, 0xc03242c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc03518a500, {0xc03242c000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc03518a500, {0xc03242c000, 0xc032c9ba40, 0xc01b516f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc016444340, {0xc03242c000, 0xc0295b5040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0632fde60, {0xc03242c000, 0xc0a165c480, 0xc01b516d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a2a20420) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a2a20420, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0632fde60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6610 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc029ce4750, 0xc036448b40) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6697 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06055e0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0205a8800, 0xc015566000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0205a8800, {0xc015566000, 0x3c76, 0x3c76}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0205a8800, {0xc015566000, 0x3c71, 0xc020561f40}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015d9efc8, {0xc015566000, 0xc015566000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020f85800, {0xc015566000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc020153e78, {0x3544be0, 0xc020f85800}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc020153c00, {0x7f09e11af0a0, 0xc04806cc00}, 0x3c76) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc020153c00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc020153c00, {0xc0168ce000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc029a84c00, {0xc00a4b8200, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc029a84c00}, {0xc00a4b8200, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00a4b8200, 0x9, 0xc001729c80}, {0x35449e0, 0xc029a84c00}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00a4b81c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc020795f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc0205d4000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 71553 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03bfe4a20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852608 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5fbc0, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b18b2780, 0xc083380e40, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc0b18b2780, {0x35a0d00, 0xc0255bc2a0}, {0xc0014c72f8, 0x40ce34}, {0x354bea0, 0xc0879d2160}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc0b18b2780, {0x35a0d00, 0xc0255bc2a0}, {0x35c3098, 0x0}, {0x35c3098, 0xc083380e10}, 0xc0014c74e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc083380e10}, 0xc0b28ec800, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc0b18b2700, {0x35a0d00, 0xc0255bc2a0}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc0255bc2a0, {0x35a0d00, 0xc0255bc2a0}, 0xc01d8ca5a8, 0x0) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc0b18b2700, {0x35a0d00, 0xc0255bc2a0}, {0x357b3f8, 0xc083380e10}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc0b18b2700, {0x35a0d00, 0xc0255bc2a0}, {0xc02560ecd0, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc019784a80, {0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0xc02560ecb0}, {0xc02fdca108, 0xc024d8a790}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0x3}, {0xc02fdca108, 0x11}, {0xc017c69860, 0x0}, 0xc019563e40) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0x3}, {0xc02fdca108, 0x11}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc0255bc2a0, {0x35a0d00, 0xc0255bc2a0}, {0x2c655e4, 0x0}, {0xc02fdca108, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f52280, {0x35a0d00, 0xc0255bc2a0}, {{}, 0x0, {0xc03c825e00, 0x5}, {0xc02fdca108, 0x11}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x6ce8c6, 0xc023ff06e0) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 10850304 [select]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08bc75380, 0xc05c5fcd00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc06edc17a0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 10843193 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc05a0d6f48, 0x2) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc05a0d6f30, {0xc09a00ec01, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc09a00ec01, 0xc09a00ec00, 0xc003ad3cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0b3b6ba40) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0b3b6ba40) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0b3b6ba40, {0x2748340, 0xc01f755290}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08da26a80, {0xc091877400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05aababe0, 0xc0f2947ef0, {0x357ac00, 0xc01f73c900}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05b016720) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc05b000e00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 9720078 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e05bb8c0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc111f1fa80, 0xc015ee0d00, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc111f1fa80, {0xc015ee0d00, 0x817, 0x817}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc111f1fa80, {0xc015ee0d00, 0x7d7, 0xc02ea603a0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc039952ee8, {0xc015ee0d00, 0xc015ee0d3b, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc08a069ae8, {0xc015ee0d00, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc01692f078, {0x3544be0, 0xc08a069ae8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc01692ee00, {0x7f09e11af0a0, 0xc02ea68580}, 0x7dc) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc01692ee00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc01692ee00, {0xc0a1265000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc02eb0cc60, {0xc042226e40, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc02eb0cc60}, {0xc042226e40, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc042226e40, 0x9, 0xc0012e6690}, {0x35449e0, 0xc02eb0cc60}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc042226e00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e457f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc02e17ea80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10847529 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc03f82c480, 0xc06e1feb00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc0b0b8a000) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 6760 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03ebf99e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10845453 [sync.Cond.Wait, 1 minutes]: | |
| sync.runtime_notifyListWait(0xc0a96d96c8, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01a90fba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc0a96d96b0, {0xc051ab1e00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc003a1c4d8}, {0xc051ab1e00, 0xc003a1c560, 0xc003a1c4f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc09f196780) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc09f196780) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc09f196780, {0x2748340, 0xc03da7cdb0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0bdcc5aa0, {0xc092e45000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00e02ebe0, 0xc0135d1b40, {0x357ac00, 0xc053739f00}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0b9c6aa40) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc053739ec0) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 8550 [select]: | |
| net/http.(*persistConn).writeLoop(0xc054db66c0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 93344 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc020bd0680, 0xc05238e440) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 6198356 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01dcd4410, 0xc0bdb0dd40) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 93450 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0730938, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0d80a3900, 0xc0db988000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0d80a3900, {0xc0db988000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0d80a3900, {0xc0db988000, 0xc033082b40, 0xc0a1199f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc036eaf6f8, {0xc0db988000, 0xc0291e6680, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0acc8dd40, {0xc0db988000, 0xc0d80ccde0, 0xc0a1199d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0db953200) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0db953200, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0acc8dd40) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10736531 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3100, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0c9457180, 0xc0a55bb000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0c9457180, {0xc0a55bb000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0c9457180, {0xc0a55bb000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc015e94fa8, {0xc0a55bb000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc0ad137800, {0xc0a55bb000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc0a2141f80) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a2141f80, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc07a10c8c0, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 94467 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0220be270, 0xc0e21d3200) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 53017 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06f730000, 0xc00a7e9cc0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 66952 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc10cae29c0, 0xc01e597860) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68277 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d16080, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc108eb9480, 0xc0fc525000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc108eb9480, {0xc0fc525000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc108eb9480, {0xc0fc525000, 0xc033086140, 0xc0713c2f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc050c74e88, {0xc0fc525000, 0xc02fbe0820, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0b7512120, {0xc0fc525000, 0xc108ed8300, 0xc0713c2d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc108eb51a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc108eb51a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0b7512120) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 3956707 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0043c12b0, 0xc0866079e0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 70181 [select]: | |
| net/http.(*persistConn).writeLoop(0xc002477c20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10806260 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc05a6cc680, 0xc058caca40) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68049 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc077ecc340, 0xc08b4c4580) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10836277 [select, 4 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc02f995e00, 0xc091ff0f00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce866, 0xc009aa6a20) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 34295 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733ec8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc079659400, 0xc00117a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc079659400, {0xc00117a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc079659400, {0xc00117a000, 0x4d9ca6, 0x4d8e460}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc013aca400, {0xc00117a000, 0x0, 0x0}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc055518990, {0xc00117a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/net/http/server.go:780 +0x16d | |
| bufio.(*Reader).fill(0xc0006cc060) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0006cc060, 0x4) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*conn).serve(0xc0550d72c0, {0x35a0d38, 0xc0011cc0c0}) | |
| /usr/local/go/src/net/http/server.go:1955 +0xc36 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3034 +0x4e8 | |
| goroutine 67236 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06ef14a20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10850276 [select]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc08bc75200, 0xc058c57400) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc07d8dc900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 9712352 [select]: | |
| net/http.(*persistConn).writeLoop(0xc08f027680) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10843875 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc063ba6780, 0xc0630e1500) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x6ce8c6, 0xc010930120) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 68052 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a6898, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0ebeec300, 0xc0ec93e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0ebeec300, {0xc0ec93e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0ebeec300, {0xc0ec93e000, 0xc032f53400, 0xc0713c4f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04fe56318, {0xc0ec93e000, 0xc02b05b860, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0923fec60, {0xc0ec93e000, 0xc0d5733bc0, 0xc0713c4d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0e547af60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0e547af60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0923fec60) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10825443 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc02b0207c8, 0x6) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xa8) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc02b0207b0, {0xc0e4fe2601, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa9}, {0xc0e4fe2601, 0xc0e4fe2600, 0xc008d3fcf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc0764c6b40) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc0764c6b40) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc0764c6b40, {0x2748340, 0xc08ee91fc8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc07aab6210, {0xc06d343c00, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc02b04a230, 0xc0013786c0, {0x357ac00, 0xc01c857c00}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc02b0315c0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02b040740) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 4048494 [select]: | |
| net/http.(*persistConn).writeLoop(0xc021b3db00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 68278 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0b7512120) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 9720838 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06444e0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0319d3480, 0xc00eaf6000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0319d3480, {0xc00eaf6000, 0x3c74, 0x3c74}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0319d3480, {0xc00eaf6000, 0x3c6f, 0xc0451e4ca0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0effbefb8, {0xc00eaf6000, 0xc00eaf6000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc01f6e34e8, {0xc00eaf6000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc03eec4cf8, {0x3544be0, 0xc01f6e34e8}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc03eec4a80, {0x7f09e11af0a0, 0xc0451e1c80}, 0x3c74) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc03eec4a80, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc03eec4a80, {0xc016644000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0453ce960, {0xc03b214c80, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0453ce960}, {0xc03b214c80, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc03b214c80, 0x9, 0xc0014c8900}, {0x35449e0, 0xc0453ce960}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc03b214c40) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e5daf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc045346780) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 10852808 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 9718815 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc08d5a0000, 0xc0ace01bc0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10841654 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc01c550648, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc01e121ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc01c550630, {0xc024bfb801, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc024bfb801, 0xc07866a800, 0xc01e121cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc088e12dc0) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc088e12dc0) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc088e12dc0, {0x2748340, 0xc0d9a825b8}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc08f6bd530, {0xc07d9b3800, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01c56e140, 0xc028c98090, {0x357ac00, 0xc023a54240}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc01c2c53e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc01c2c1c00) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 68697 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0732aa0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a1218e80, 0xc067d1e000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a1218e80, {0xc067d1e000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a1218e80, {0xc067d1e000, 0xc033082dc0, 0xc0229a2f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc050c75b68, {0xc067d1e000, 0xc0148f6d00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08e4fa480, {0xc067d1e000, 0xc03cab43c0, 0xc0229a2d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc003b68120) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc003b68120, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08e4fa480) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 69226 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0576bc000) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 139898 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02c12d200) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 8496 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3670, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc077b80400, 0xc077283000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc077b80400, {0xc077283000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc077b80400, {0xc077283000, 0xc033083a40, 0xc01a25cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017e6f150, {0xc077283000, 0xc019d61520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc051bb6fc0, {0xc077283000, 0xc0c7b13a40, 0xc01a25cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0776dd440) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0776dd440, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc051bb6fc0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 4048493 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f820, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0197d1d00, 0xc093296000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0197d1d00, {0xc093296000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0197d1d00, {0xc093296000, 0xc032c9be00, 0xc01e45bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc08b18e350, {0xc093296000, 0xc05e524680, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc021b3db00, {0xc093296000, 0xc08ed44420, 0xc01e45bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0f95404e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0f95404e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc021b3db00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67518 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b210, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0c9456c80, 0xc0c9d3a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0c9456c80, {0xc0c9d3a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0c9456c80, {0xc0c9d3a000, 0xc032f58c80, 0xc020816f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04d6ab240, {0xc0c9d3a000, 0xc0407c7a00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc02b30cd80, {0xc0c9d3a000, 0xc00440ba40, 0xc020816d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0c7f62ae0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0c7f62ae0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc02b30cd80) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 90851 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc02939d2b0, 0xc007edd7a0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10841822 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc02cdea4c8, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc00e519ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc02cdea4b0, {0xc0f71ad801, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xac}, {0xc0f71ad801, 0xc0fa9ec400, 0xc00e519cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc08445ac80) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc08445ac80) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc08445ac80, {0x2748340, 0xc09e444630}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc05a40aa80, {0xc01d4f3c00, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc02cdfa0a0, 0xc01b3a6bd0, {0x357ac00, 0xc0ca8fc400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc02cd58440) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02ccddc80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 8549 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733de0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc077b80880, 0xc077c23000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc077b80880, {0xc077c23000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc077b80880, {0xc077c23000, 0xc033083180, 0xc01e5d5f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017e6f190, {0xc077c23000, 0xc019d5dd40, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc054db66c0, {0xc077c23000, 0xc0c7a8f320, 0xc01e5d5d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0776dd7a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0776dd7a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc054db66c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10845452 [select, 1 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc0a96d9680, 0xc0af33c800) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x29, 0xc0889571c0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 8545 [select]: | |
| net/http.(*persistConn).writeLoop(0xc051bb6fc0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 69250 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0442a3320) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10834103 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc089237e48, 0x0) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc089237e38) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc089237e30, {0xc05cf57c00, 0x200, 0x200}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xc006bc14d8}, {0xc05cf57c00, 0xc006bc1560, 0xc006bc14f0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc043f19040) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc043f19040) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc043f19040, {0x2748340, 0xc07b9f51d0}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc01f66c150, {0xc0b5fd1000, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01feef040, 0xc0137892c0, {0x357ac00, 0xc0be1d2ec0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc058465760) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0be1d2e80) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 67793 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09bbd04e0, 0xc0197c1a00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 98581 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d24e30, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0bfbe8f00, 0xc0c091f000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0bfbe8f00, {0xc0c091f000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0bfbe8f00, {0xc0c091f000, 0xc032c9ba40, 0xc01a13df68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc03d7269e0, {0xc0c091f000, 0xc0280c5ba0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc032655b00, {0xc0c091f000, 0xc0f0845d40, 0xc01a13dd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0c09184e0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0c09184e0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc032655b00) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 70180 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e07329b8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0db3a1900, 0xc08ffcb000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0db3a1900, {0xc08ffcb000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0db3a1900, {0xc08ffcb000, 0xc032f59540, 0xc01cb57f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04bbee3e8, {0xc08ffcb000, 0xc036d8d380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc002477c20, {0xc08ffcb000, 0xc0bfa9b680, 0xc01cb57d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc063e36f60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc063e36f60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc002477c20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 92509 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15f98, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a2964c80, 0xc0a318b000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a2964c80, {0xc0a318b000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a2964c80, {0xc0a318b000, 0xc032f53680, 0xc01ac73f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc056187ce0, {0xc0a318b000, 0xc02b05b1e0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc017694fc0, {0xc0a318b000, 0xc0b8007b60, 0xc01ac73d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a318df80) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a318df80, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc017694fc0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 6205049 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc010e71040, 0xc0af4026c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 8126 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03e7edc20) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 93678 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0929dc7e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10852712 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0733fb0, 0x77) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc01cb65380, 0xc03c268de0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:93 | |
| internal/poll.(*FD).WaitWrite(...) | |
| /usr/local/go/src/internal/poll/fd_unix.go:529 | |
| net.(*netFD).connect(0xc01cb65380, {0x35a0d00, 0xc02552ec60}, {0xc0171a32f8, 0x40ce34}, {0x354bea0, 0xc0b35b1300}) | |
| /usr/local/go/src/net/fd_unix.go:142 +0x717 | |
| net.(*netFD).dial(0xc01cb65380, {0x35a0d00, 0xc02552ec60}, {0x35c3098, 0x0}, {0x35c3098, 0xc03c268db0}, 0xc0171a34e8) | |
| /usr/local/go/src/net/sock_posix.go:150 +0x379 | |
| net.socket({0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0x3}, 0x2, 0x1, 0x0, 0xe8, {0x35c3098, 0x0}, ...) | |
| /usr/local/go/src/net/sock_posix.go:71 +0x2a5 | |
| net.internetSocket({0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0x3}, {0x35c3098, 0x0}, {0x35c3098, 0xc03c268db0}, 0xc08bc9fda0, 0x0, ...) | |
| /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 | |
| net.(*sysDialer).doDialTCP(0xc01cb65300, {0x35a0d00, 0xc02552ec60}, 0x0, 0x27310a0) | |
| /usr/local/go/src/net/tcpsock_posix.go:66 +0xa5 | |
| net.(*sysDialer).dialTCP(0xc02552ec60, {0x35a0d00, 0xc02552ec60}, 0xc01d8ca5a8, 0x0) | |
| /usr/local/go/src/net/tcpsock_posix.go:62 +0x59 | |
| net.(*sysDialer).dialSingle(0xc01cb65300, {0x35a0d00, 0xc02552ec60}, {0x357b3f8, 0xc03c268db0}) | |
| /usr/local/go/src/net/dial.go:583 +0x28b | |
| net.(*sysDialer).dialSerial(0xc01cb65300, {0x35a0d00, 0xc02552ec60}, {0xc02579c390, 0x1, 0x2c65e59}) | |
| /usr/local/go/src/net/dial.go:551 +0x312 | |
| net.(*Dialer).DialContext(0xc0197b7500, {0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0xc02579c370}, {0xc023544438, 0xc024d8b080}) | |
| /usr/local/go/src/net/dial.go:428 +0x736 | |
| github.com/mwitkow/go-conntrack.dialClientConnTracker({0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0x3}, {0xc023544438, 0x12}, {0xc017c696e0, 0x0}, 0xc0197b2ec0) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:123 +0x1ef | |
| github.com/mwitkow/go-conntrack.NewDialContextFunc.func1({0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0x3}, {0xc023544438, 0x12}) | |
| /go/pkg/mod/github.com/mwitkow/go-conntrack@v0.0.0-20190716064945-2f068394615f/dialer_wrapper.go:94 +0xf5 | |
| net/http.(*Transport).dial(0xc02552ec60, {0x35a0d00, 0xc02552ec60}, {0x2c655e4, 0x0}, {0xc023544438, 0x40ce34}) | |
| /usr/local/go/src/net/http/transport.go:1166 +0xda | |
| net/http.(*Transport).dialConn(0xc032f59e00, {0x35a0d00, 0xc02552ec60}, {{}, 0x0, {0xc03692a000, 0x5}, {0xc023544438, 0x12}, 0x0}) | |
| /usr/local/go/src/net/http/transport.go:1604 +0x845 | |
| net/http.(*Transport).dialConnFor(0x35a0d38, 0xc0ecafc210) | |
| /usr/local/go/src/net/http/transport.go:1446 +0xb0 | |
| created by net/http.(*Transport).queueForDial | |
| /usr/local/go/src/net/http/transport.go:1415 +0x3d7 | |
| goroutine 66948 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc10cae2340, 0xc0650e01c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 91330 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a0ffe8f0, 0xc03a0ee340) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 69613 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d15bf8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0290b2a00, 0xc0eb734000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0290b2a00, {0xc0eb734000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0290b2a00, {0xc0eb734000, 0xc032f53680, 0xc02e504f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04a5825a8, {0xc0eb734000, 0xc0371ce1a0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0682eab40, {0xc0eb734000, 0xc09ec3b440, 0xc02e504d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc023ed1440) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc023ed1440, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0682eab40) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 136262 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18ff188, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0881d0d00, 0xc056c8b000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0881d0d00, {0xc056c8b000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0881d0d00, {0xc056c8b000, 0x4c50, 0xc08876a260}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc065d2aa20, {0xc056c8b000, 0xc056c8b000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020c3cd98, {0xc056c8b000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc082dffaf8, {0x3544be0, 0xc020c3cd98}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc082dff880, {0x7f09e11af0a0, 0xc088b85e40}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc082dff880, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc082dff880, {0xc0c9ab2000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc077bd7860, {0xc00fdc6ac0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc077bd7860}, {0xc00fdc6ac0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc00fdc6ac0, 0x9, 0xc0015fb080}, {0x35449e0, 0xc077bd7860}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00fdc6a80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0a119df98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc087629080) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 67113 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0228ab860, 0xc0f4d22240) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852721 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 6199301 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644d08, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc07a464f80, 0xc0aeb8a000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc07a464f80, {0xc0aeb8a000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc07a464f80, {0xc0aeb8a000, 0xc032f59a40, 0xc01bb9ff68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0236de498, {0xc0aeb8a000, 0xc036760340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0602ba900, {0xc0aeb8a000, 0xc10ba85500, 0xc01bb9fd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc078d85c80) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc078d85c80, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0602ba900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 92419 [select]: | |
| net/http.(*persistConn).writeLoop(0xc045c62ea0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 54108 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0648c30e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 8125 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734180, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc074cfe600, 0xc074f7c000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc074cfe600, {0xc074f7c000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc074cfe600, {0xc074f7c000, 0xc032c9bb80, 0xc01988cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc017ea4368, {0xc074f7c000, 0xc019d5d040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc03e7edc20, {0xc074f7c000, 0xc0a48d3800, 0xc01988cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc074de5b00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc074de5b00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc03e7edc20) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10843192 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc05a0d6f00, 0xc05a845900) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc08da265d0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 68042 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc075ca3450, 0x4dbf460) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 3959472 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0234281a0, 0xc0844c3000) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 91347 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01b2751e0, 0xc01802eb00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10852907 [select]: | |
| net/http.(*persistConn).writeLoop(0xc0162eb560) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10603718 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b128, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0558dda00, 0xc037afa000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0558dda00, {0xc037afa000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0558dda00, {0xc037afa000, 0x4c50, 0xc05380e2a0}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00c8671b8, {0xc037afa000, 0xc037afa000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020c3cea0, {0xc037afa000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc049661078, {0x3544be0, 0xc020c3cea0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc049660e00, {0x7f09e11af0a0, 0xc05382c080}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc049660e00, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc049660e00, {0xc0a2e9b000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc053845bc0, {0xc086e199a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc053845bc0}, {0xc086e199a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc086e199a0, 0x9, 0xc0015fb440}, {0x35449e0, 0xc053845bc0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc086e19960) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01e603f98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc045b62d80) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 9713358 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b868, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0205a9a00, 0xc06f2bf000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0205a9a00, {0xc06f2bf000, 0x4c55, 0x4c55}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0205a9a00, {0xc06f2bf000, 0x4c50, 0xc0f8604e60}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc048dc03c0, {0xc06f2bf000, 0xc06f2bf000, 0x5}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| crypto/tls.(*atLeastReader).Read(0xc020f18bd0, {0xc06f2bf000, 0x0, 0x409b6d}) | |
| /usr/local/go/src/crypto/tls/conn.go:777 +0x3d | |
| bytes.(*Buffer).ReadFrom(0xc06666e278, {0x3544be0, 0xc020f18bd0}) | |
| /usr/local/go/src/bytes/buffer.go:204 +0x98 | |
| crypto/tls.(*Conn).readFromUntil(0xc06666e000, {0x7f09e11af0a0, 0xc0658c2700}, 0x4c55) | |
| /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 | |
| crypto/tls.(*Conn).readRecordOrCCS(0xc06666e000, 0x0) | |
| /usr/local/go/src/crypto/tls/conn.go:606 +0x112 | |
| crypto/tls.(*Conn).readRecord(...) | |
| /usr/local/go/src/crypto/tls/conn.go:574 | |
| crypto/tls.(*Conn).Read(0xc06666e000, {0xc0ba455000, 0x1000, 0x8d4340}) | |
| /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f | |
| bufio.(*Reader).Read(0xc0eec7cea0, {0xc047f704a0, 0x9, 0x8e2302}) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x1b4 | |
| io.ReadAtLeast({0x35449e0, 0xc0eec7cea0}, {0xc047f704a0, 0x9, 0x9}, 0x9) | |
| /usr/local/go/src/io/io.go:328 +0x9a | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader({0xc047f704a0, 0x9, 0xc001386030}, {0x35449e0, 0xc0eec7cea0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc047f70460) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/frame.go:498 +0x95 | |
| golang.org/x/net/http2.(*clientConnReadLoop).run(0xc01caccf98) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2101 +0x130 | |
| golang.org/x/net/http2.(*ClientConn).readLoop(0xc06eb47200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1997 +0x6f | |
| created by golang.org/x/net/http2.(*Transport).newClientConn | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:725 +0xac5 | |
| goroutine 135271 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e06054f8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0efb7b680, 0xc0dce9b000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0efb7b680, {0xc0dce9b000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0efb7b680, {0xc0dce9b000, 0xc032f58780, 0xc01e5d7f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0610c7778, {0xc0dce9b000, 0xc006f78680, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0403d5320, {0xc0dce9b000, 0xc02b2b94a0, 0xc01e5d7d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0b3919860) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0b3919860, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0403d5320) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 92895 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc091946820, 0xc0b58878c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10851852 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0734520, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0aba77f00, 0xc0bea2d000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0aba77f00, {0xc0bea2d000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0aba77f00, {0xc0bea2d000, 0xc033083e00, 0xc003a32f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc03a5a2510, {0xc0bea2d000, 0xc0400ab520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc0866ca120, {0xc0bea2d000, 0xc08f4d6960, 0xc003a32d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc01ecea900) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc01ecea900, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc0866ca120) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 90844 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc048e6c340, 0xc0197c16c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 92382 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605b50, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc09ebe3d80, 0xc0a28c0000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc09ebe3d80, {0xc0a28c0000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc09ebe3d80, {0xc0a28c0000, 0xc033082dc0, 0xc01e5def68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc05513f348, {0xc0a28c0000, 0xc019d60340, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05583f8c0, {0xc0a28c0000, 0xc0be9851a0, 0xc01e5ded30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a287b500) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a287b500, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05583f8c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 68043 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc075ca3520, 0xc0b3408900) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 93398 [select]: | |
| net/http.(*persistConn).writeLoop(0xc03d6a10e0) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 10603130 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f0a8bd5f398, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0b443e500, 0xc0e1aef000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0b443e500, {0xc0e1aef000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0b443e500, {0xc0e1aef000, 0xc032f58780, 0xc0713c5f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0df1e1828, {0xc0e1aef000, 0xc0148f7040, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc126c8e900, {0xc0e1aef000, 0xc03dfc25a0, 0xc0713c5d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc06ec0d4a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc06ec0d4a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc126c8e900) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10819705 [select, 8 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc00e614f00, 0xc0540ee200) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc06f4260f0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 10841653 [select, 3 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc01c550600, 0xc0482a9300) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc08f6bc930) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 10852550 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e18feb30, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a80ce300, 0xc03bcba000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a80ce300, {0xc03bcba000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a80ce300, {0xc03bcba000, 0xc032f58b40, 0xc02e506f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00673c628, {0xc03bcba000, 0xc023976000, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc00f3959e0, {0xc03bcba000, 0xc031152c60, 0xc02e506d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc020bdf800) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc020bdf800, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc00f3959e0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 91628 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0908ce8f0, 0xc0388890c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 95454 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0644df0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc094a8c900, 0xc0d0d4f000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc094a8c900, {0xc0d0d4f000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc094a8c900, {0xc0d0d4f000, 0xc032f59cc0, 0xc02008cf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc037878d38, {0xc0d0d4f000, 0xc053e6f6c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc027e44000, {0xc0d0d4f000, 0xc0250f3680, 0xc02008cd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc075f8b320) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc075f8b320, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc027e44000) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10818887 [select, 8 minutes]: | |
| golang.org/x/net/http2.(*clientStream).writeRequest(0xc00d446180, 0xc066bb3c00) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1323 +0xaae | |
| golang.org/x/net/http2.(*clientStream).doRequest(0x35a1588, 0xc03f34d6b0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1185 +0x1e | |
| created by golang.org/x/net/http2.(*ClientConn).RoundTrip | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:1114 +0x30f | |
| goroutine 67906 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b3be0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0e10a1a00, 0xc0dcc4d000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0e10a1a00, {0xc0dcc4d000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0e10a1a00, {0xc0dcc4d000, 0xc033083cc0, 0xc020850f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc04ec0f798, {0xc0dcc4d000, 0xc02f7a36c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc094141d40, {0xc0dcc4d000, 0xc0f0a6b200, 0xc020850d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0e22e7e00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0e22e7e00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc094141d40) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 53830 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0d2b3e0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a76b9800, 0xc0a770f000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a76b9800, {0xc0a770f000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a76b9800, {0xc0a770f000, 0xc033083040, 0xc00073bf68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc043604f78, {0xc0a770f000, 0xc0a6cd9a00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc05f4f78c0, {0xc0a770f000, 0xc02513dc20, 0xc00073bd30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0a77c2120) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0a77c2120, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc05f4f78c0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 53016 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc06fdd7ee0, 0xc036cec840) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 4051601 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11b4ff0, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0844ebf00, 0xc07a569000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0844ebf00, {0xc07a569000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0844ebf00, {0xc07a569000, 0xc032c9be00, 0xc01c7a3f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc00692dfa0, {0xc07a569000, 0xc02c640d00, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc086682360, {0xc07a569000, 0xc0457bd5c0, 0xc01c7a3d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc08617cf00) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc08617cf00, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc086682360) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 67591 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc09e9c08f0, 0xc0e90a42c0) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 68393 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0748c8680, 0xc0e5a06b00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 91280 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0a2096dd0, 0xc01802fb00) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 67519 [select]: | |
| net/http.(*persistConn).writeLoop(0xc02b30cd80) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 4051119 [select]: | |
| github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc07b5ba750, 0xc051c5f320) | |
| /app/scrape/scrape.go:1205 +0x425 | |
| created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync | |
| /app/scrape/scrape.go:584 +0xa55 | |
| goroutine 10790982 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e0605328, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc06e35ec80, 0xc06e404000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc06e35ec80, {0xc06e404000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc06e35ec80, {0xc06e404000, 0xc032f59b80, 0xc020790f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc027e2a8e8, {0xc06e404000, 0xc01376d6c0, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08e43f320, {0xc06e404000, 0xc08cbf4fc0, 0xc020790d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc06e2645a0) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc06e2645a0, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08e43f320) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852948 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 98582 [select]: | |
| net/http.(*persistConn).writeLoop(0xc032655b00) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 | |
| goroutine 68065 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a66c8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0eceaa080, 0xc0edc42000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0eceaa080, {0xc0edc42000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0eceaa080, {0xc0edc42000, 0xc033083b80, 0xc01e5d1f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0504304b0, {0xc0edc42000, 0xc0400ab380, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc09ba505a0, {0xc0edc42000, 0xc0ecc51320, 0xc01e5d1d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc0ecf40c60) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc0ecf40c60, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc09ba505a0) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10847451 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc04094a048, 0x1) | |
| /usr/local/go/src/runtime/sema.go:513 +0x13d | |
| sync.(*Cond).Wait(0xc008d43ba0) | |
| /usr/local/go/src/sync/cond.go:56 +0x8c | |
| golang.org/x/net/http2.(*pipe).Read(0xc04094a030, {0xc0e3878001, 0x5ff, 0x5ff}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/pipe.go:76 +0xeb | |
| golang.org/x/net/http2.transportResponseBody.Read({0xa8}, {0xc0e3878001, 0xc0474e3200, 0xc008d43cf0}) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20211123203042-d83791d6bcd9/http2/transport.go:2384 +0x85 | |
| encoding/json.(*Decoder).refill(0xc036b9c140) | |
| /usr/local/go/src/encoding/json/stream.go:165 +0x17f | |
| encoding/json.(*Decoder).readValue(0xc036b9c140) | |
| /usr/local/go/src/encoding/json/stream.go:140 +0xbb | |
| encoding/json.(*Decoder).Decode(0xc036b9c140, {0x2748340, 0xc01f59b398}) | |
| /usr/local/go/src/encoding/json/stream.go:63 +0x78 | |
| k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc03c97d530, {0xc01ed4a400, 0x400, 0x400}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/util/framer/framer.go:152 +0x19c | |
| k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0409461e0, 0xc054db27e0, {0x357ac00, 0xc01e17cdc0}) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 | |
| k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0406768e0) | |
| /go/pkg/mod/k8s.io/client-go@v0.22.4/rest/watch/decoder.go:49 +0x4f | |
| k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc040750440) | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:105 +0x107 | |
| created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher | |
| /go/pkg/mod/k8s.io/apimachinery@v0.22.4/pkg/watch/streamwatcher.go:76 +0x135 | |
| goroutine 68732 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f09e11a5be8, 0x72) | |
| /usr/local/go/src/runtime/netpoll.go:234 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0a2c70500, 0xc016734000, 0x0) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0a2c70500, {0xc016734000, 0x1000, 0x1000}) | |
| /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0a2c70500, {0xc016734000, 0xc033083cc0, 0xc003a30f68}) | |
| /usr/local/go/src/net/fd_posix.go:56 +0x29 | |
| net.(*conn).Read(0xc0333ccd58, {0xc016734000, 0xc0148fb520, 0x4}) | |
| /usr/local/go/src/net/net.go:183 +0x45 | |
| net/http.(*persistConn).Read(0xc08e50b560, {0xc016734000, 0xc06b31e900, 0xc003a30d30}) | |
| /usr/local/go/src/net/http/transport.go:1926 +0x4e | |
| bufio.(*Reader).fill(0xc003a7f080) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x103 | |
| bufio.(*Reader).Peek(0xc003a7f080, 0x1) | |
| /usr/local/go/src/bufio/bufio.go:139 +0x5d | |
| net/http.(*persistConn).readLoop(0xc08e50b560) | |
| /usr/local/go/src/net/http/transport.go:2087 +0x1ac | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1747 +0x1e05 | |
| goroutine 10852905 [select]: | |
| net.(*netFD).connect.func2() | |
| /usr/local/go/src/net/fd_unix.go:119 +0x9e | |
| created by net.(*netFD).connect | |
| /usr/local/go/src/net/fd_unix.go:118 +0x385 | |
| goroutine 67234 [select]: | |
| net/http.(*persistConn).writeLoop(0xc06ef14900) | |
| /usr/local/go/src/net/http/transport.go:2386 +0xfb | |
| created by net/http.(*Transport).dialConn | |
| /usr/local/go/src/net/http/transport.go:1748 +0x1e65 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment