2.17.1 memory leak goroutine dump

goroutine 4091033 [running]:
runtime/pprof.writeGoroutineStacks(0x7f4ed5aa1730, 0xc0462442d0, 0x0, 0x0)
	/usr/local/go/src/runtime/pprof/pprof.go:679 +0x9d
runtime/pprof.writeGoroutine(0x7f4ed5aa1730, 0xc0462442d0, 0x2, 0x40c1d6, 0xc03bcc71e0)
	/usr/local/go/src/runtime/pprof/pprof.go:668 +0x44
runtime/pprof.(*Profile).WriteTo(0x4099880, 0x7f4ed5aa1730, 0xc0462442d0, 0x2, 0xc0462442d0, 0xc1fe91b238)
	/usr/local/go/src/runtime/pprof/pprof.go:329 +0x3da
net/http/pprof.handler.ServeHTTP(0xc00271120d, 0x9, 0x7f4ed5aa1660, 0xc0462442d0, 0xc0731fa300)
	/usr/local/go/src/net/http/pprof/pprof.go:245 +0x33a
net/http/pprof.Index(0x7f4ed5aa1660, 0xc0462442d0, 0xc0731fa300)
	/usr/local/go/src/net/http/pprof/pprof.go:268 +0x6f1
github.com/prometheus/prometheus/web.serveDebug(0x7f4ed5aa1660, 0xc0462442d0, 0xc0731fa300)
	/app/web/web.go:470 +0x1cf
net/http.HandlerFunc.ServeHTTP(0x2a5df00, 0x7f4ed5aa1660, 0xc0462442d0, 0xc0731fa300)
	/usr/local/go/src/net/http/server.go:2007 +0x44
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1(0x7f4ed5aa1660, 0xc046244280, 0xc0731fa300)
	/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196 +0xe9
net/http.HandlerFunc.ServeHTTP(0xc000217c80, 0x7f4ed5aa1660, 0xc046244280, 0xc0731fa300)
	/usr/local/go/src/net/http/server.go:2007 +0x44
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2(0x7f4ed5aa1660, 0xc046244280, 0xc0731fa300)
	/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76 +0xb2
net/http.HandlerFunc.ServeHTTP(0xc000217d70, 0x7f4ed5aa1660, 0xc046244280, 0xc0731fa300)
	/usr/local/go/src/net/http/server.go:2007 +0x44
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1(0x2e13e80, 0xc046244230, 0xc0731fa300)
	/app/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100 +0xda
github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1(0x2e13e80, 0xc046244230, 0xc04b01ff00)
	/app/web/web.go:1122 +0x1c6
github.com/prometheus/common/route.(*Router).handle.func1(0x2e13e80, 0xc046244230, 0xc04b01fe00, 0xc04641c740, 0x1, 0x1)
	/app/vendor/github.com/prometheus/common/route/route.go:83 +0x296
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc00049c900, 0x2e13e80, 0xc046244230, 0xc04b01fe00)
	/app/vendor/github.com/julienschmidt/httprouter/router.go:334 +0x95e
github.com/prometheus/common/route.(*Router).ServeHTTP(0xc0006d1800, 0x2e13e80, 0xc046244230, 0xc04b01fe00)
	/app/vendor/github.com/prometheus/common/route/route.go:121 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc000112680, 0x2e13e80, 0xc046244230, 0xc04b01fe00)
	/usr/local/go/src/net/http/server.go:2387 +0x1bd
github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5(0x2e12480, 0xc09909f260, 0xc311023900)
	/app/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:140 +0x5fb
net/http.HandlerFunc.ServeHTTP(0xc00075f9e0, 0x2e12480, 0xc09909f260, 0xc311023900)
	/usr/local/go/src/net/http/server.go:2007 +0x44
github.com/prometheus/prometheus/web.withStackTracer.func1(0x2e12480, 0xc09909f260, 0xc311023900)
	/app/web/web.go:103 +0xb3
net/http.HandlerFunc.ServeHTTP(0xc00075fa10, 0x2e12480, 0xc09909f260, 0xc311023900)
	/usr/local/go/src/net/http/server.go:2007 +0x44
net/http.serverHandler.ServeHTTP(0xc00076e000, 0x2e12480, 0xc09909f260, 0xc311023900)
	/usr/local/go/src/net/http/server.go:2802 +0xa4
net/http.(*conn).serve(0xc046248140, 0x2e1c480, 0xc00b0285c0)
	/usr/local/go/src/net/http/server.go:1890 +0x875
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 1 [chan receive, 4874 minutes]:
github.com/oklog/run.(*Group).Run(0xc000571c08, 0xc0007a8230, 0x8)
	/app/vendor/github.com/oklog/run/group.go:43 +0xed
main.main()
	/app/cmd/prometheus/main.go:746 +0x6ec0

goroutine 17 [syscall, 4874 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 8 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000ca820)
	/app/vendor/go.opencensus.io/stats/view/worker.go:154 +0x100
created by go.opencensus.io/stats/view.init.0
	/app/vendor/go.opencensus.io/stats/view/worker.go:32 +0x57

goroutine 55 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc000441860)
	/app/discovery/manager.go:263 +0x123
created by github.com/prometheus/prometheus/discovery.(*Manager).Run
	/app/discovery/manager.go:170 +0x43

goroutine 118 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).sender(0xc000441720)
	/app/discovery/manager.go:263 +0x123
created by github.com/prometheus/prometheus/discovery.(*Manager).Run
	/app/discovery/manager.go:170 +0x43

goroutine 135 [select]:
github.com/prometheus/prometheus/pkg/logging.(*Deduper).run(0xc00049c380)
	/app/pkg/logging/dedupe.go:75 +0x1f3
created by github.com/prometheus/prometheus/pkg/logging.Dedupe
	/app/pkg/logging/dedupe.go:61 +0xcf

goroutine 155 [chan receive]:
github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run(0xc00072ef00)
	/app/storage/remote/write.go:86 +0xc7
created by github.com/prometheus/prometheus/storage/remote.NewWriteStorage
	/app/storage/remote/write.go:79 +0x1a9

goroutine 92 [select, 4874 minutes]:
main.main.func6(0x0, 0x0)
	/app/cmd/prometheus/main.go:510 +0xfa
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc000226c00, 0xc0007a8150)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 93 [chan receive, 4874 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc000441720, 0x0, 0x0)
	/app/discovery/manager.go:171 +0x72
main.main.func8(0x0, 0x0)
	/app/cmd/prometheus/main.go:530 +0x4e
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc0007a0460, 0xc0007a0480)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 94 [chan receive, 4874 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc000441860, 0x0, 0x0)
	/app/discovery/manager.go:171 +0x72
main.main.func10(0x0, 0x0)
	/app/cmd/prometheus/main.go:544 +0x4e
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc0007a04c0, 0xc0007a04e0)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 95 [select]:
github.com/prometheus/prometheus/scrape.(*Manager).Run(0xc00008fda0, 0xc0005fe240, 0x0, 0x0)
	/app/scrape/manager.go:142 +0xdc
main.main.func12(0x0, 0x0)
	/app/cmd/prometheus/main.go:564 +0x8b
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc00078c1b0, 0xc0007a0500)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 96 [select, 778 minutes]:
main.main.func14(0x0, 0x0)
	/app/cmd/prometheus/main.go:589 +0x16d
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc000102420, 0xc0007a81a0)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 193 [chan receive, 4874 minutes]:
main.main.func16(0x0, 0x0)
	/app/cmd/prometheus/main.go:636 +0x398
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc000102480, 0xc0007a81b0)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 194 [chan receive, 4874 minutes]:
main.main.func18(0x0, 0x0)
	/app/cmd/prometheus/main.go:652 +0x72
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc0007a0520, 0xc0007a0540)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 195 [chan receive, 4874 minutes]:
main.main.func20(0x6, 0x8)
	/app/cmd/prometheus/main.go:699 +0x989
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc000226c80, 0xc00078c1e0)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 196 [select, 4874 minutes]:
github.com/prometheus/prometheus/web.(*Handler).Run(0xc000333200, 0x2e1c480, 0xc00049c440, 0x6, 0x8)
	/app/web/web.go:591 +0x1039
main.main.func22(0x6, 0x8)
	/app/cmd/prometheus/main.go:714 +0x40
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc0007a0560, 0xc0007a8220)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 197 [select]:
github.com/prometheus/prometheus/notifier.(*Manager).Run(0xc00072f280, 0xc0005fe300)
	/app/notifier/notifier.go:308 +0xfd
main.main.func24(0x6, 0x8)
	/app/cmd/prometheus/main.go:737 +0x8b
github.com/oklog/run.(*Group).Run.func1(0xc0001024e0, 0xc00078c210, 0xc0007a8230)
	/app/vendor/github.com/oklog/run/group.go:38 +0x27
created by github.com/oklog/run.(*Group).Run
	/app/vendor/github.com/oklog/run/group.go:37 +0xbb

goroutine 225 [select, 84 minutes]:
github.com/prometheus/prometheus/tsdb/wal.(*WAL).run(0xc000460750)
	/app/tsdb/wal/wal.go:321 +0xdb
created by github.com/prometheus/prometheus/tsdb/wal.NewSize
	/app/tsdb/wal/wal.go:290 +0x33a

goroutine 119 [select, 4874 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000112600)
	/app/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69 +0xc2
created by google.golang.org/grpc.newCCBalancerWrapper
	/app/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:60 +0x16d

goroutine 120 [chan receive, 4874 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0000e4000)
	/app/vendor/google.golang.org/grpc/clientconn.go:1179 +0x6ea
created by google.golang.org/grpc.(*addrConn).connect
	/app/vendor/google.golang.org/grpc/clientconn.go:815 +0x128

goroutine 121 [chan receive, 4874 minutes]:
github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1.1(0x2e1c480, 0xc00049c440, 0xc0005b7180, 0x26266ff, 0xc)
	/app/prompb/rpc.pb.gw.go:202 +0x4c
created by github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1
	/app/prompb/rpc.pb.gw.go:201 +0x1b4

goroutine 58 [IO wait, 4874 minutes]:
internal/poll.runtime_pollWait(0x7f4ed5b63448, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000636118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000636100, 0xc000d38000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000636100, 0xc000d38000, 0x8000, 0x8000, 0x4, 0xc000d54ce0, 0x4053cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000600008, 0xc000d38000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
bufio.(*Reader).Read(0xc0006442a0, 0xc000356038, 0x9, 0x9, 0xc000d54d98, 0x406134, 0xc0006164e0)
	/usr/local/go/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x2dca380, 0xc0006442a0, 0xc000356038, 0x9, 0x9, 0x9, 0xc000132190, 0x21f4800, 0xc00061c3c0)
	/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000356038, 0x9, 0x9, 0x2dca380, 0xc0006442a0, 0x0, 0xc000000000, 0xc00061c3c0, 0xc00010ac88)
	/app/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000356000, 0xc000436f30, 0x1, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00015e380)
	/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1265 +0x183
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:300 +0xd23

goroutine 122 [chan receive]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:229
net/http.(*Server).Serve(0xc00076e000, 0x2e13340, 0xc00010c440, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2896 +0x280
github.com/prometheus/prometheus/web.(*Handler).Run.func3(0xc00076a600, 0xc00076e000, 0x2e13340, 0xc00010c440)
	/app/web/web.go:582 +0x3f
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/app/web/web.go:581 +0xef9

goroutine 123 [chan receive, 4874 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:229
google.golang.org/grpc.(*Server).Serve(0xc000564480, 0x2e13340, 0xc00010c240, 0x0, 0x0)
	/app/vendor/google.golang.org/grpc/server.go:597 +0x22e
github.com/prometheus/prometheus/web.(*Handler).Run.func4(0xc00076a600, 0xc000564480, 0x2e13340, 0xc00010c240)
	/app/web/web.go:585 +0x3f
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/app/web/web.go:584 +0xf42

goroutine 124 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b63518, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0005d4318, 0x72, 0x0, 0x0, 0x261ee6b)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0005d4300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0005d4300, 0x1, 0xc000186600, 0x100000001)
	/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00010c120, 0xc00057e060, 0xc000bb4da0, 0x2)
	/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00010c120, 0x1, 0xc04699b140, 0x23a3740, 0x23a3740)
	/usr/local/go/src/net/tcpsock.go:261 +0x47
golang.org/x/net/netutil.(*limitListener).Accept(0xc0005cc1b0, 0xc04699b110, 0xc0005cc1e0, 0x2e47800, 0x438a7e)
	/app/vendor/golang.org/x/net/netutil/listen.go:48 +0x51
github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc00010c200, 0x2a5dfa8, 0xc00061e190, 0x2e47800, 0xc04699b140)
	/app/vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go:100 +0x7f
github.com/soheilhy/cmux.(*cMux).Serve(0xc00061e190, 0x0, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:162 +0xaf
github.com/prometheus/prometheus/web.(*Handler).Run.func5(0xc00076a600, 0x2e2e0c0, 0xc00061e190)
	/app/web/web.go:588 +0x31
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/app/web/web.go:587 +0xf7e

goroutine 125 [IO wait, 4874 minutes]:
internal/poll.runtime_pollWait(0x7f4ed5b63378, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0005d4a98, 0x72, 0x0, 0x9, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0005d4a80, 0xc00076e118, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0005d4a80, 0xc00076e118, 0x9, 0x9, 0x9, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000138fa8, 0xc00076e118, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc000322010, 0xc00076e118, 0x9, 0x9, 0xc000098000, 0x7f4ed7f70460, 0x0)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
io.ReadAtLeast(0x2dcbe60, 0xc000322010, 0xc00076e118, 0x9, 0x9, 0x9, 0xecb898, 0xc00074c2d0, 0xc000740004)
	/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00076e118, 0x9, 0x9, 0x2dcbe60, 0xc000322010, 0x0, 0x0, 0xc00074c2d0, 0x0)
	/app/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00076e0e0, 0x2de2d40, 0xc00074c2d0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/soheilhy/cmux.matchHTTP2Field(0x7f4ed5b26a18, 0xc00075fad0, 0x2dcbe60, 0xc000322010, 0x26276bf, 0xc, 0xc000816ce0, 0x40a1b5)
	/app/vendor/github.com/soheilhy/cmux/matchers.go:236 +0x148
github.com/soheilhy/cmux.HTTP2MatchHeaderFieldSendSettings.func1(0x7f4ed5b26a18, 0xc00075fad0, 0x2dcbe60, 0xc000322010, 0xc00075fad0)
	/app/vendor/github.com/soheilhy/cmux/matchers.go:173 +0xca
github.com/soheilhy/cmux.(*cMux).serve(0xc00061e190, 0x2e47800, 0xc00075fad0, 0xc00057e120, 0xc0006cae60)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:184 +0x1ee
created by github.com/soheilhy/cmux.(*cMux).Serve
	/app/vendor/github.com/soheilhy/cmux/cmux.go:171 +0x1a3

goroutine 59 [select, 4874 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00014a0f0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000644360, 0x0, 0x0)
	/app/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1e3
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00015e380)
	/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/app/vendor/google.golang.org/grpc/internal/transport/http2_client.go:344 +0xedb

goroutine 1326 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faaa00, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4091034 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d721e8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0470e3598, 0x72, 0x0, 0x1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0470e3580, 0xc046246881, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0470e3580, 0xc046246881, 0x1, 0x1, 0x1575fcc, 0xc04119f180, 0x44e528)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009259f28, 0xc046246881, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc046990470, 0xc046246881, 0x1, 0x1, 0x40c0d60, 0x12f2e6c, 0x2a5f0b0)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc046990460, 0xc046246881, 0x1, 0x1, 0x16, 0x20, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).backgroundRead(0xc046246870)
	/usr/local/go/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:673 +0xd4

goroutine 4088886 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc011ee17a0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc011ee1790)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc011ee1788, 0xc0012b1e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc011ee1760, 0xc0012b1e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc011ee1a20, 0xc0328f05c0, 0xc0f2ebc201)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc011ee1a20, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc011ee1a20, 0x223e960, 0xc0328f05c0, 0x203044, 0x203044)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc042d177a0, 0xc12e7c6800, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc013846280, 0x0, 0x2de7300, 0xc1321f1440, 0xc04d0aa300, 0x2a5fc88, 0x40a2bf0, 0xc0021fe650, 0xc0021fe750)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0328f05a0, 0x70500d, 0x70242c, 0xc011f0f900, 0x2e1c500, 0xc0138443c0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc1321f1400)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3444495 [select]:
net/http.(*persistConn).writeLoop(0xc0d285c120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439370 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4400)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c1b0, 0xc0021f6370)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 274 [select, 2 minutes]:
github.com/prometheus/prometheus/tsdb.(*DB).run(0xc0002f2dd0)
	/app/tsdb/db.go:647 +0x203
created by github.com/prometheus/prometheus/tsdb.open
	/app/tsdb/db.go:614 +0x653

goroutine 3439365 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c230, 0xc01588a2a0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc284094320, 0xc360366080)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3443996 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2eaa0, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439191 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc1340b0398)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1340b0390)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b4ea0, 0x2e1c480, 0xc0100d0400, 0xc2d1d5f680)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 1812 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244b40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1814 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244c80, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1566 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0027697c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439308 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001728fd0, 0x2de7680, 0xc0c42e3000, 0xc000e87bf8, 0xc03be7cde0, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001728fd0, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01e82eef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000e87ef0, 0x3b9aca00, 0x0, 0xc001b2af01, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001728fd0, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0a30, 0xc0b1e552e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439192 [select, 2 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441860, 0x2e1c480, 0xc0100d0400, 0xc03c484550, 0xc2d1d5f680)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 9413 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc018e205a0, 0x2540be400, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1638 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999cc0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3433955 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f3d0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc138f7f018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc138f7f000, 0xc13d260000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc138f7f000, 0xc13d260000, 0x1000, 0x1000, 0xc00105e140, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00c4f3530, 0xc13d260000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc13989d200, 0xc13d260000, 0x1000, 0x1000, 0xc02fbbb680, 0xc001c20c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc13d1c84e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc13d1c84e0, 0x1, 0x0, 0x0, 0x1, 0xc2b6553c00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc13989d200)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1639 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999e00, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439352 [chan receive, 6 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x40618a, 0xc2b5e78780, 0x40)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc059427dd8, 0xc001780360, 0xc001a09d88)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001a09f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059427f40, 0xdf8475800, 0x0, 0x42eb01, 0xc045b31500)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4380)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c220, 0xc017d24e80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1640 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999ea0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439381 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a20, 0xc0d8359900)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 1357629 [select]:
net/http.(*persistConn).writeLoop(0xc04f08e7e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4082751 [select]:
net/http.(*persistConn).writeLoop(0xc12e07fd40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1635 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999900, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3298 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143ce0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143ce0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4091075 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4ebb0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc05658e798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc05658e780, 0xc0464c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc05658e780, 0xc0464c2000, 0x1000, 0x1000, 0xc011f0f900, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0154c70d8, 0xc0464c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc039d39b00, 0xc0464c2000, 0x1000, 0x1000, 0xc052de4780, 0xc001766c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc045e2e240)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc045e2e240, 0x1, 0x0, 0x0, 0x1, 0xc052de4600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc039d39b00)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1921 [select]:
net/http.(*persistConn).writeLoop(0xc00b535680)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1634 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0019997c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1636 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999a40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357783 [select]:
net/http.(*persistConn).writeLoop(0xc12006a120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357905 [select]:
net/http.(*persistConn).writeLoop(0xc0b4160b40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4087982 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0c52c3e80, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0c52c3e70)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0c52c3e68, 0xc05a9c5400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0c52c3e40, 0xc05a9c5400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0bddf1ce0, 0xc0c8670ac0, 0x6001)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0bddf1ce0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0bddf1ce0, 0x223e960, 0xc0c8670ac0, 0xc00181b598, 0x0)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0c3ed8870, 0xc0ba3f1000, 0x400, 0x400, 0x0, 0xc075468b40, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0c8722280, 0x0, 0x2de7300, 0xc0c42e3040, 0x21f589af00000000, 0x5e95aace, 0xc00181b658, 0x486d66, 0x5e95aace)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0c8670aa0, 0xffffffffffffff01, 0xc00181b6c8, 0x0, 0xc075468b40, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0c42e3000)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1637 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc001999b80, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439396 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc05830f180)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a7480, 0xc017d251c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1805 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9950, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001b1f298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b1f280, 0xc00615a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001b1f280, 0xc00615a000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc005262058, 0xc00615a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00b1fe240, 0xc00615a000, 0x1000, 0x1000, 0xc005eba9c0, 0xc001922c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc006158000)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc006158000, 0x1, 0x0, 0x0, 0x1, 0xc032366500, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00b1fe240)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1702 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0010185a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439596 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0029a2090, 0xc2000000b4)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0029a2080)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc03be7c3c0, 0x0, 0x0, 0xc07a611f00)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0ddeb4300, 0x2e1c480, 0xc009bf0b00, 0xc2d1d5e600, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0ddeb4300, 0x2e1c480, 0xc009bf0b00, 0xc2d1d5e600)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3439064 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc03be22360)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 1806 [select]:
net/http.(*persistConn).writeLoop(0xc00b1fe240)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1813 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244be0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1564 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002769680, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3086485 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a80c8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc20b9d7898, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc20b9d7880, 0xc124500000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc20b9d7880, 0xc124500000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc003ced620, 0xc124500000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc04d7e7fc0, 0xc124500000, 0x3823, 0x3823, 0x28, 0x8, 0xc01ea658a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0d83fa258, 0x2dca560, 0xc04d7e7fc0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0d83fa000, 0x7f4ed5b388a0, 0xc01020e380, 0x5, 0xc01020e380, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0d83fa000, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0d83fa000, 0xc01b25c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc12c684fc0, 0xc01b25c000, 0x1000, 0x1000, 0xc02d127f80, 0xc01ea65c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0d83fc840)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0d83fc840, 0x1, 0x0, 0x0, 0x1, 0xc00b2ae600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc12c684fc0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1736 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc000f3e140, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1978 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9470, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00f750598, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00f750580, 0xc00faac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00f750580, 0xc00faac000, 0x1000, 0x1000, 0xc14bfe3a01, 0x10000c14bfe3a80, 0x40c5ec0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f7d2050, 0xc00faac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc001846080, 0xc00faac000, 0x1000, 0x1000, 0x72, 0x0, 0xc00fa7e188)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc001846070, 0xc00faac000, 0x1000, 0x1000, 0x10a430f81f979, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc00fa7e180, 0xc00faac000, 0x1000, 0x1000, 0xc14bfe3c38, 0x1e8711a, 0xc00f7d66c0)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc00fa7c0c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00fa7c0c0, 0x4, 0x10a430f81f979, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc00faaa000, 0x2e1c480, 0xc00fa96040)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 1920 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b628e8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00b231e98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00b231e80, 0xc00b732000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00b231e80, 0xc00b732000, 0x1000, 0x1000, 0xc00105ec80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00b62e808, 0xc00b732000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00b535680, 0xc00b732000, 0x1000, 0x1000, 0xc003d23c20, 0xc000d73c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00b63f380)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00b63f380, 0x1, 0x0, 0x0, 0x1, 0xc03ac13700, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00b535680)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1568 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0032448c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1942 [select]:
net/http.(*persistConn).writeLoop(0xc00f65e120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439301 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b498230, 0xc03e31aae0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0860, 0xc0b1e54c40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 171594 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa65a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 5349 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62818, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001d88a98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001d88a80, 0xc001789000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001d88a80, 0xc001789000, 0x1000, 0x1000, 0xc0044d43c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0136c4430, 0xc001789000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00f863e60, 0xc001789000, 0x1000, 0x1000, 0xc01305e0c0, 0xc01a20ec20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc010770720)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc010770720, 0x1, 0x0, 0x0, 0x1, 0xc03ac12f00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00f863e60)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439341 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0501778c0, 0x2de7680, 0xc13ea3b100, 0xc26be09bf8, 0xc0154133e0, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0501778c0, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc074267ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc26be09ef0, 0x3b9aca00, 0x0, 0xc00170df01, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0501778c0, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2bf0, 0xc35fd5c3e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439748 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00fea3890, 0xc000010efd)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00fea3880)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc03be22360, 0x0, 0x0, 0xc046246700)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc05830f080, 0x2e1c480, 0xc009bf0900, 0xc2d1d5e360, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc05830f080, 0x2e1c480, 0xc009bf0900, 0xc2d1d5e360)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3439068 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc05830f080, 0x2e1c480, 0xc009bf0900, 0xc2d1d5e360)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc1383857c0, 0x2e1c480, 0xc009bf0900, 0xc2d1d5e360, 0x2dcba20, 0xc05830f080)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3439393 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc023d55edc, 0x2, 0x8)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc001893dd8, 0x0, 0xc05187e740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc023d55f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001893f40, 0xdf8475800, 0x0, 0x42eb01, 0xc045b31740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0d8358000)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4980d0, 0xc017d25190)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4088537 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc14094f900, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc14094f8f0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc14094f8e8, 0xc007784c00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc14094f8c0, 0xc007784c00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc14094fb80, 0xc14494a580, 0x2a5fa01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc14094fb80, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc14094fb80, 0x223e960, 0xc14494a580, 0xc048a79d98, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc1449b8630, 0xc10a978c00, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc13abcae60, 0x0, 0x2de7300, 0xc143042880, 0x21a8ff12004313fc, 0x5e95aaec, 0xc048a79e58, 0x486d66, 0x5e95aaec)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc14494a560, 0xc048a79ee8, 0x3, 0x0, 0xc0b9464000, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc143042840)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1968 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9540, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00f750518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00f750500, 0xc00faa8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00f750500, 0xc00faa8000, 0x1000, 0x1000, 0xc002963a40, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f956308, 0xc00faa8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00f7717a0, 0xc00faa8000, 0x1000, 0x1000, 0xc00f8b74a0, 0xc005163c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00fa3a120)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00fa3a120, 0x1, 0x0, 0x0, 0x1, 0xc005a17300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00f7717a0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1809 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244960, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357640 [select]:
net/http.(*persistConn).writeLoop(0xc00c27eb40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4088852 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc011e01380, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc011e01370)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc011e01368, 0xc0012b0000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc011e01340, 0xc0012b0000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc011e01600, 0xc0a43e4260, 0xc001836c01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc011e01600, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc011e01600, 0x223e960, 0xc0a43e4260, 0x203003, 0x203003)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc014d611d0, 0xc12e7c6400, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc012f95cc0, 0x0, 0x2de7300, 0xc051c34740, 0x21a3a28001d905a0, 0x5e95ab00, 0xc002203e58, 0x486d66, 0x5e95ab00)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0a43e4240, 0x8, 0xc002203e50, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc051c34700)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 9384 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc01883e280, 0x2540be400, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4090625 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc012715220, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc012715210)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc012715208, 0xc093117400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0127151e0, 0xc093117400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0127154a0, 0xc066ad9d80, 0xc0d1d54101)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0127154a0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0127154a0, 0x223e960, 0xc066ad9d80, 0x18, 0x40ca48)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc051948390, 0xc0ca62d800, 0x400, 0x400, 0x0, 0x70, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc013939310, 0x0, 0x2de7300, 0xc2f47263c0, 0x21cde15e1c7d2000, 0x5e95ab6e, 0xc023e06658, 0x486d66, 0x5e95ab6e)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc066ad9d60, 0x70500d, 0x70242c, 0xc011f0f900, 0x2e1c500, 0xc013936180, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc2f4726380)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1810 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244a00, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4088293 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed53a7978, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc10561f318, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc10561f300, 0xc02f2bc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc10561f300, 0xc02f2bc000, 0x1000, 0x1000, 0xc11196fa01, 0x10000c11196fa80, 0x40c60c0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00ffa0e98, 0xc02f2bc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc10f732160, 0xc02f2bc000, 0x1000, 0x1000, 0x72, 0x0, 0xc10f470ea8)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc10f732150, 0xc02f2bc000, 0x1000, 0x1000, 0x10a296c7cc3c4, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc10f470ea0, 0xc02f2bc000, 0x1000, 0x1000, 0xc11196fc38, 0x1e8711a, 0xc10f470e40)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc0262d9a40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0262d9a40, 0x4, 0x10a296c7cc3c4, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc10e6c65a0, 0x2e1c480, 0xc1fbcf5440)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 3439377 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc001735f38, 0x3, 0x3)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0086bbdd8, 0x10000000044d5bb, 0x14)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001735f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0086bbf40, 0xdf8475800, 0x0, 0x42eb01, 0xc0522e7740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc05830f300)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a7640, 0xc02cc010e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3433674 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc1204e94a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439403 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30d80, 0xc0ddeb5a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439183 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4600)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c300, 0xc002606470)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4089406 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc07e198b40, 0x91)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc07e198b30)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc07e198b28, 0xc047455201, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc07e198b00, 0xc047455201, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc07e198dc0, 0xc04641c60a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc07e198dc0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc07e198dc0, 0x223e960, 0xc04641c660, 0x0, 0x0)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc074e7b7d0, 0xc00bdb4400, 0x400, 0x400, 0x2de4dc0, 0x0, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc07da59310, 0x0, 0x2de7300, 0xc00b028580, 0x0, 0x0, 0x0, 0xc05346be58, 0x4590b0)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc07e4be380, 0xc13e549fa8, 0x8, 0x2de4dc0, 0xc00bd07040, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc02aa618c0)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1560 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002769400, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1324 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faa8c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 164866 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc04cf2ad20, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc04cf2ad20)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1811 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003244aa0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439433 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc045c4cdc0, 0xc029a3b6e0, 0xc2dde42b40, 0xc032f796e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3425688 [select]:
net/http.(*persistConn).writeLoop(0xc2efefbd40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3440185 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0029a2150, 0xc1000000b3)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0029a2140)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc03be7c480, 0x0, 0x0, 0xc0996a1900)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0ddeb4500, 0x2e1c480, 0xc009bf0b40, 0xc2d1d5e660, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0ddeb4500, 0x2e1c480, 0xc009bf0b40, 0xc2d1d5e660)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 4090811 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71dd8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001ae0598, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001ae0580, 0xc032a8a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001ae0580, 0xc032a8a000, 0x1000, 0x1000, 0xc011f0f900, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0154c6d58, 0xc032a8a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00430cea0, 0xc032a8a000, 0x1000, 0x1000, 0xc005a17140, 0xc001c21c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc02dd855c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc02dd855c0, 0x1, 0x0, 0x0, 0x1, 0xc005a17000, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00430cea0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3102 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb810a0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb810a0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1784 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002f0d0e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357527 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d72118, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc04c56e218, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc04c56e200, 0xc0a0ffc000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc04c56e200, 0xc0a0ffc000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025ee50, 0xc0a0ffc000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc058d750c0, 0xc0a0ffc000, 0x3823, 0x3823, 0x28, 0x8, 0xc0154398a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0991d7058, 0x2dca560, 0xc058d750c0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0991d6e00, 0x7f4ed5b388a0, 0xc00c4bac40, 0x5, 0xc00c4bac40, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0991d6e00, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0991d6e00, 0xc0a0fe8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0054c19e0, 0xc0a0fe8000, 0x1000, 0x1000, 0xc029f524e0, 0xc015439c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a0e53c80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a0e53c80, 0x1, 0x0, 0x0, 0x1, 0xc03ac13600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0054c19e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 5522 [select]:
net/http.(*persistConn).writeLoop(0xc00fd58a20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1705 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcdb80, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1325 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faa960, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1708 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003176aa0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1042 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10540, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10540)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439221 [sync.Cond.Wait, 778 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0017290a8, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc001729098)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001729080, 0xc002168270, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc126a7eb80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01e829e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01e829e00, 0x3b9aca00, 0x0, 0xc03c2e4401, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc126a7eb80, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122990, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 1939 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b96e0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00f64e218, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00f64e200, 0xc00df97000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00f64e200, 0xc00df97000, 0x1000, 0x1000, 0xc00105ec80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00bd5e348, 0xc00df97000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00f65e000, 0xc00df97000, 0x1000, 0x1000, 0xc00f65c240, 0xc000d75c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00bd69560)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00bd69560, 0x1, 0x0, 0x0, 0x1, 0xc032367100, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00f65e000)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439270 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b498380, 0xc0522e7020)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2630, 0xc01a111aa0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3177679 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc002b9e7e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc002b9e7e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439238 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049f38398, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049f38388)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc049f38370, 0xc001a9dc10, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002515c80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01af9fe00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01af9fe00, 0x3b9aca00, 0x0, 0xc26dd72401, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002515c80, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e1b0, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3965248 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62a88, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc01220bb18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc01220bb00, 0xc0325ce000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc01220bb00, 0xc0325ce000, 0x1000, 0x1000, 0xc002963a40, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc013804048, 0xc0325ce000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0030cea20, 0xc0325ce000, 0x1000, 0x1000, 0xc136463c80, 0xc001765c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc02b319620)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc02b319620, 0x1, 0x0, 0x0, 0x1, 0xc2b5379100, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0030cea20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439280 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4983f0, 0xc0522e7320)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2a10, 0xc35fd5c020)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1561 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0027694a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357671 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9130, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002579718, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002579700, 0xc0a39f4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002579700, 0xc0a39f4000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc014541fe0, 0xc0a39f4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0026559e0, 0xc0a39f4000, 0x1000, 0x1000, 0xc0333de0c0, 0xc0641b9c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a39bea20)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a39bea20, 0x1, 0x0, 0x0, 0x1, 0xc00b2af600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0026559e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1041 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc001590460, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc001590460)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1642 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faf180, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3438970 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc031eebda0)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 3439359 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x100c023e08db8, 0x40)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc059426dd8, 0x44d9d7, 0x40c5e40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc023e08f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059426f40, 0xdf8475800, 0x0, 0x46d801, 0xc045b316e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc05830f380)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a75d0, 0xc017d25170)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439237 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0179184f8, 0x68)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0179184e8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0179184d0, 0xc017d24ba0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb5280)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc059421e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059421e00, 0x3b9aca00, 0x0, 0xc003928501, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb5280, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e240, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 1563 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0027695e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1782 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcbcc0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 171590 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa6320, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1567 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002fa3d60, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439350 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc058761601, 0x100000040c1d6, 0x40)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc01a215dd8, 0x4, 0x438a7e)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc058761740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01a215f40, 0xdf8475800, 0x0, 0x42eb01, 0xc045b314a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4200)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c060, 0xc017d24e60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439394 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0d8358000)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4980d0, 0xc017d251a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357716 [select]:
net/http.(*persistConn).writeLoop(0xc002655560)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 992 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00076e1c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00076e1c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1783 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002f0d040, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1778 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcba40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 5457 [select]:
net/http.(*persistConn).writeLoop(0xc00c0acea0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357662 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d718f8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d8358298, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d8358280, 0xc0a3a60000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d8358280, 0xc0a3a60000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009982d40, 0xc0a3a60000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc0e19d2420, 0xc0a3a60000, 0x3823, 0x3823, 0x28, 0x8, 0xc0154358a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a2b81058, 0x2dca560, 0xc0e19d2420, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a2b80e00, 0x7f4ed5b388a0, 0xc02ce4bec0, 0x5, 0xc02ce4bec0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a2b80e00, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a2b80e00, 0xc0a3a5e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc04f204240, 0xc0a3a5e000, 0x1000, 0x1000, 0xc02cc2c0c0, 0xc015435c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a3a5c1e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a3a5c1e0, 0x1, 0x0, 0x0, 0x1, 0xc2c0f57800, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f204240)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3438974 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc002514e80, 0x2e1c480, 0xc009bf08c0, 0xc2d1d5e300)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc13a3cdd50, 0x2e1c480, 0xc009bf08c0, 0xc2d1d5e300, 0x2dcba20, 0xc002514e80)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 5324 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc016ca3900, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3424378 [select, 786 minutes]:
github.com/prometheus/prometheus/storage/remote.(*shards).enqueue(0xc12032b8c0, 0x823, 0xc31809a900, 0x17, 0x18, 0x17175d741e9, 0x41b4d0ea61000000, 0xc11db1c000)
	/app/storage/remote/queue_manager.go:752 +0x187
github.com/prometheus/prometheus/storage/remote.(*QueueManager).Append(0xc120132800, 0xc11db1c000, 0x11ef, 0x1400, 0x1001)
	/app/storage/remote/queue_manager.go:336 +0x497
github.com/prometheus/prometheus/tsdb/wal.(*Watcher).readSegment(0xc0b9c7b7a0, 0xc2cfa9a000, 0x44, 0x1, 0x2dca701, 0xc000082110)
	/app/tsdb/wal/watcher.go:505 +0x311
github.com/prometheus/prometheus/tsdb/wal.(*Watcher).watch(0xc0b9c7b7a0, 0x44, 0x1, 0x0, 0x0)
	/app/tsdb/wal/watcher.go:410 +0x3f5
github.com/prometheus/prometheus/tsdb/wal.(*Watcher).Run(0xc0b9c7b7a0, 0xdedddb1ca845, 0x40c0d60)
	/app/tsdb/wal/watcher.go:250 +0x6a7
github.com/prometheus/prometheus/tsdb/wal.(*Watcher).loop(0xc0b9c7b7a0)
	/app/tsdb/wal/watcher.go:199 +0x13a
created by github.com/prometheus/prometheus/tsdb/wal.(*Watcher).Start
	/app/tsdb/wal/watcher.go:173 +0x192

goroutine 3439397 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x1f4)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc003fb5dd8, 0x0, 0xc0c3e445f8)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0c3e44740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003fb5f40, 0xdf8475800, 0x0, 0x42eb01, 0xc045b31800)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0d8358200)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498220, 0xc017d251d0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439360 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc05830f380)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a75d0, 0xc017d25180)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439265 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4980e0, 0xc0522e6ea0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d24a0, 0xc01a111620)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439404 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc017918840, 0x2de7680, 0xc008d7ec80, 0xc05c7bbbf8, 0xc03d0104e0, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc017918840, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00187aef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc05c7bbef0, 0x3b9aca00, 0x0, 0xc001739f01, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc017918840, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920890, 0xc352eb8280)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439203 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a80, 0xc002515480)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439407 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc001728fd0, 0xc0522e6a80, 0xc045b31c20, 0xc03be7cde0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439351 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4200)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c060, 0xc017d24e70)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439267 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0501771e0, 0x2de7680, 0xc0f47e8340, 0xc0016ebbf8, 0xc0255241e0, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0501771e0, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc05c7bbef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0016ebef0, 0x3b9aca00, 0x0, 0xc02e3dcf01, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0501771e0, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d24e0, 0xc01a111740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439249 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc017918658, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc017918648)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc017918630, 0xc017d24eb0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb5680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc059425e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc059425e00, 0x3b9aca00, 0x0, 0xc26d8f4d01, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb5680, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc001814120, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3439311 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4982a0, 0xc03e31ade0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0b70, 0xc0b1e55ec0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1707 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003176a00, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4086016 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc017918580, 0xc045b30a80, 0xc032f3c4e0, 0xc08d3fc2a0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439275 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b498310, 0xc0522e71a0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d27c0, 0xc01a111f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439251 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc05830f280, 0x2e1c480, 0xc009bf0940, 0xc2d1d5e3c0)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc138385800, 0x2e1c480, 0xc009bf0940, 0xc2d1d5e3c0, 0x2dcba20, 0xc05830f280)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3444485 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ece0d3800, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0b5960518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0b5960500, 0xc0b5b52000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0b5960500, 0xc0b5b52000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc013804440, 0xc0b5b52000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0b5962240, 0xc0b5b52000, 0x1000, 0x1000, 0xc012537d40, 0xc01a20fc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0b4350cc0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0b4350cc0, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2400, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b5962240)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3435682 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc1230f08c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc1230f08c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 171592 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa6460, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1045 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000357c00, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000357c00)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1562 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002769540, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4087746 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc05ba314e0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc05ba314d0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc05ba314c8, 0xc01da12e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc05ba314a0, 0xc01da12e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc05ba31760, 0xc05bc60e00, 0xc00170d401)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc05ba31760, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc05ba31760, 0x223e960, 0xc05bc60e00, 0xc03f730240, 0x4)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc05ce14210, 0xc035605000, 0x400, 0x400, 0x0, 0x2, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05b7dca50, 0x0, 0x2de7300, 0xc008764900, 0x21dc1290040de4e0, 0x5e95aab5, 0xc00170d658, 0x486d66, 0x5e95aab5)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05bc60de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0087648c0)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439239 [sync.Cond.Wait, 10 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049f384f8, 0x208)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049f384e8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc049f384d0, 0xc001a9df50, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0e1598080)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000bc9e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000bc9e00, 0x3b9aca00, 0x0, 0xc03c30e401, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0e1598080, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e2d0, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 3439702 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc002c68650, 0xc100016939)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc002c68640)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc0343c55c0, 0x0, 0x0, 0xc04699b200)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0d8358380, 0x2e1c480, 0xc009bf0a40, 0xc2d1d5e540, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0d8358380, 0x2e1c480, 0xc009bf0a40, 0xc2d1d5e540)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 1367 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002e25220, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439330 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc050177760, 0x2de7680, 0xc002900e00, 0xc1fe91dbf8, 0xc03fff0660, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc050177760, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc005185ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1fe91def0, 0x3b9aca00, 0x0, 0xc000393701, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc050177760, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2a50, 0xc35fd5c0e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439295 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc017918420, 0x2de7680, 0xc0c08ca540, 0xc1fe913bf8, 0xc03e23f980, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc017918420, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001c24ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1fe913ef0, 0x3b9aca00, 0x0, 0xc023d52701, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc017918420, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920410, 0xc02c80d600)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 183825 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0541127e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0541127e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1819 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b629b8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0022ced18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0022ced00, 0xc002a89000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0022ced00, 0xc002a89000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0016446f8, 0xc002a89000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00211ea20, 0xc002a89000, 0x1000, 0x1000, 0xc0015e2d20, 0xc002821c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00078e780)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00078e780, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2800, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00211ea20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1641 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faf0e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439214 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c0d0, 0xc001a9dbf0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1820 [select]:
net/http.(*persistConn).writeLoop(0xc00211ea20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439260 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a7420, 0xc3548d11a0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc1383859a0, 0xc0e0a0bf00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3424376 [sleep]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
time.Sleep(0x3b9aca00)
	/usr/local/go/src/runtime/time.go:105 +0x157
github.com/prometheus/prometheus/storage/remote.(*shards).sendSamplesWithBackoff(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0xc03608c000, 0xa5f, 0x1388, 0xc2fa84fea8, 0x7b1140, 0xc00049c380)
	/app/storage/remote/queue_manager.go:886 +0x13b
github.com/prometheus/prometheus/storage/remote.(*shards).sendSamples(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0xc03608c000, 0xa5f, 0x1388, 0xc01a211ea8)
	/app/storage/remote/queue_manager.go:838 +0xb3
github.com/prometheus/prometheus/storage/remote.(*shards).runShard(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0x0, 0xc04ff64420)
	/app/storage/remote/queue_manager.go:827 +0xa98
created by github.com/prometheus/prometheus/storage/remote.(*shards).start
	/app/storage/remote/queue_manager.go:704 +0x272

goroutine 4069348 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f230, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc09d82ee98, 0x72, 0x3c00, 0x3ca1, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc09d82ee80, 0xc0512f2000, 0x3ca1, 0x3ca1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc09d82ee80, 0xc0512f2000, 0x3ca1, 0x3ca1, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f51e0c8, 0xc0512f2000, 0x3ca1, 0x3ca1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc0b27dc940, 0xc0512f2000, 0x3ca1, 0x3ca1, 0x28, 0x8, 0xc000d528a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc04a4673d8, 0x2dca560, 0xc0b27dc940, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc04a467180, 0x7f4ed5b388a0, 0xc010901580, 0x5, 0xc010901580, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc04a467180, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc04a467180, 0xc04f82c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0144278c0, 0xc04f82c000, 0x1000, 0x1000, 0xc031a2c960, 0xc000d52c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc04f82a120)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc04f82a120, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0144278c0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3445185 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ece0d3a70, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002947d98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002947d80, 0xc07abc9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002947d80, 0xc07abc9000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00ffa1280, 0xc07abc9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0036f4b40, 0xc07abc9000, 0x1000, 0x1000, 0xc2bed4df20, 0xc0641b4c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc02f189260)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc02f189260, 0x1, 0x0, 0x0, 0x1, 0xc053cc9100, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0036f4b40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1365 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00197f0e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 170721 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc04d8ff260, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc04d8ff260)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1706 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003176960, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357669 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b92d0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d779bb98, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d779bb80, 0xc0a39c6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d779bb80, 0xc0a39c6000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc014541fc8, 0xc0a39c6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc1327ec800, 0xc0a39c6000, 0x3823, 0x3823, 0x28, 0x8, 0xc0051648a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0987785d8, 0x2dca560, 0xc1327ec800, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc098778380, 0x7f4ed5b388a0, 0xc004baad00, 0x5, 0xc004baad00, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc098778380, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc098778380, 0xc0a39c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc05204cc60, 0xc0a39c4000, 0x1000, 0x1000, 0xc053b9fc20, 0xc005164c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a39be8a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a39be8a0, 0x1, 0x0, 0x0, 0x1, 0xc2f6907d00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc05204cc60)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439444 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0179186e0, 0xc3548d0a20, 0xc019346120, 0xc03c489e60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 4090039 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0e69fb4e0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0e69fb4d0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0e69fb4c8, 0xc0195b7400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0e69fb4a0, 0xc0195b7400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0e69fb760, 0xc0e6337440, 0x1)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0e69fb760, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0e69fb760, 0x223e960, 0xc0e6337440, 0x203046, 0x203046)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0e735c180, 0xc060802400, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0e6331b30, 0x0, 0x2de7300, 0xc1fbfdc980, 0x21e23290004313fc, 0x5e95ab50, 0xc0caa4be58, 0x486d66, 0x5e95ab50)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0e6337420, 0xc00072f280, 0x2e1c500, 0xc097504720, 0xc0494ee870, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc1fbfdc940)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 986 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000357ce0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000357ce0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1779 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcbae0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439336 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc001a06f38, 0x3, 0x3)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00281add8, 0x10000000044d5bb, 0xd)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001a06f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00281af40, 0xdf8475800, 0x0, 0x42eb01, 0xc0522e7500)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0d8358480)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498370, 0xc02cc00dd0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3444915 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0210c8fa0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3425519 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0375dc6e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3444981 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed4d72048, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc1231bcc18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc1231bcc00, 0xc001de0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc1231bcc00, 0xc001de0000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0070a0510, 0xc001de0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc029042b40, 0xc001de0000, 0x1000, 0x1000, 0xc2bb43e4e0, 0xc01ea67c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc011e18840)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc011e18840, 0x1, 0x0, 0x0, 0x1, 0xc053d36a00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc029042b40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439078 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc032f78ea0)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 1728 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcb900, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439334 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x5, 0x2000, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00310bdd8, 0xc012daf080, 0x122ae5f)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001694f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00310bf40, 0xdf8475800, 0x0, 0x42eb01, 0xc0522e74a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0d8358400)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4983e0, 0xc02cc00db0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3177832 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb800e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb800e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1357437 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9f00, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc047390518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc047390500, 0xc0930f2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc047390500, 0xc0930f2000, 0x1000, 0x1000, 0xc00105e3c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc012fb35b8, 0xc0930f2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0076027e0, 0xc0930f2000, 0x1000, 0x1000, 0xc04caaba40, 0xc00171ac20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc057c118c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc057c118c0, 0x1, 0x0, 0x0, 0x1, 0xc0015e2100, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0076027e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1781 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcbc20, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3437973 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc129f47b20, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc129f47b20)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1357739 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71758, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc11c0c6298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc11c0c6280, 0xc0a43f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc11c0c6280, 0xc0a43f6000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00fa7aa60, 0xc0a43f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0b4118120, 0xc0a43f6000, 0x1000, 0x1000, 0xc02cc2c540, 0xc000d55c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a39bf140)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a39bf140, 0x1, 0x0, 0x0, 0x1, 0xc04b916a00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b4118120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 989 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10700, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10700)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4074203 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed4d71b68, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0015a8498, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0015a8480, 0xc0a85c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0015a8480, 0xc0a85c4000, 0x1000, 0x1000, 0xc00f79fa01, 0x10000c00f79fa80, 0x40c5f40)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0016db8c0, 0xc0a85c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc05536ea20, 0xc0a85c4000, 0x1000, 0x1000, 0x72, 0x0, 0xc055e19c88)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc05536ea10, 0xc0a85c4000, 0x1000, 0x1000, 0x10a2aa5d197a1, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc055e19c80, 0xc0a85c4000, 0x1000, 0x1000, 0xc00f79fc38, 0x1e8711a, 0xc055e19c20)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc0a844eb40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a844eb40, 0x4, 0x10a2aa5d197a1, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc055091360, 0x2e1c480, 0xc004631d40)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 1366 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00197f220, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1777 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcb9a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 988 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10000, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10000)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1044 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f101c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f101c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1780 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcbb80, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3444993 [select, 2 minutes]:
net/http.(*persistConn).writeLoop(0xc0c3b23440)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1043 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000357ea0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000357ea0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439348 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc017918580, 0x2de7680, 0xc2f4726380, 0xc1fe4b5bf8, 0xc08d3fc2a0, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc017918580, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc200c0fef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1fe4b5ef0, 0x3b9aca00, 0x0, 0xc00062ef01, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc017918580, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920590, 0xc02c80da80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 990 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f107e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f107e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1047 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0003562a0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0003562a0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439409 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0017291e0, 0x2de7680, 0xc010730600, 0xc14003dbf8, 0xc03d0308a0, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0017291e0, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc003fb6ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc14003def0, 0x3b9aca00, 0x0, 0xc021d19701, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0017291e0, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0bb0, 0xc35560c360)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439526 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc017918840, 0xc045b30d80, 0xc213eb2ba0, 0xc03d0104e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 1565 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002769720, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 987 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10380, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10380)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1004 [select]:
github.com/prometheus/prometheus/scrape.(*Manager).reloader(0xc00008fda0)
	/app/scrape/manager.go:166 +0x1aa
created by github.com/prometheus/prometheus/scrape.(*Manager).Run
	/app/scrape/manager.go:140 +0x54

goroutine 1357631 [select]:
net/http.(*persistConn).writeLoop(0xc04f1d3560)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439244 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00172a5a8, 0x67)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00172a598)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00172a580, 0xc002606150, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc001f47a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00281be00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00281be00, 0x3b9aca00, 0x0, 0xc003928801, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc001f47a80, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e3f0, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 3437943 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc129f47880, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc129f47880)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4085603 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0480f5a60, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0480f5a50)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0480f5a48, 0xc0e6272000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0480f5a20, 0xc0e6272000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0480f5ce0, 0xc0481b3320, 0xc001828701)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0480f5ce0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0480f5ce0, 0x223e960, 0xc0481b3320, 0xc00181c598, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc045435530, 0xc0d51ef800, 0x400, 0x400, 0x0, 0xc010e11500, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc048349180, 0x0, 0x2de7300, 0xc00049d2c0, 0x21f90f43040de4e0, 0x5e95aa2e, 0xc00181c658, 0x486d66, 0x5e95aa2e)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0481b3300, 0x0, 0x2dccd20, 0x40906b, 0xc00162e000, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00049d280)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 985 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f102a0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f102a0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1057 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10620, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10620)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1031 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0015900e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0015900e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1032 [runnable]:
github.com/prometheus/prometheus/tsdb.(*memSeries).iterator(0xc024152270, 0x51, 0xc0a9fdd740, 0x0, 0x0, 0x0, 0x0)
	/app/tsdb/head.go:1914 +0x265
github.com/prometheus/prometheus/tsdb.(*safeChunk).Iterator(0xc13254b5c0, 0x0, 0x0, 0x1, 0xc046bb0de0)
	/app/tsdb/head.go:1330 +0x6d
github.com/prometheus/prometheus/tsdb.(*chunkSeriesIterator).resetCurIterator(0xc046bb0de0)
	/app/tsdb/querier.go:1069 +0x6b
github.com/prometheus/prometheus/tsdb.newChunkSeriesIterator(...)
	/app/tsdb/querier.go:1062
github.com/prometheus/prometheus/tsdb.(*chunkSeries).Iterator(0xc03e680420, 0x1, 0x1)
	/app/tsdb/querier.go:870 +0xf6
github.com/prometheus/prometheus/promql.(*evaluator).eval(0xc0013f1308, 0x2e1c980, 0xc0a9fdd500, 0xc0a9fdd9b0, 0xc0a9fdd8f0)
	/app/promql/engine.go:1114 +0x189c
github.com/prometheus/prometheus/promql.(*evaluator).rangeEval(0xc0013f1308, 0xc0013eff30, 0xc0013f0040, 0x2, 0x2, 0x0, 0x1, 0x1)
	/app/promql/engine.go:887 +0x1408
github.com/prometheus/prometheus/promql.(*evaluator).eval(0xc0013f1308, 0x2e1c900, 0xc03b91a060, 0x1, 0xc0a9fdd920)
	/app/promql/engine.go:1032 +0x99a
github.com/prometheus/prometheus/promql.(*evaluator).rangeEval(0xc0013f1308, 0xc0013f0ca8, 0xc1260aea80, 0x2, 0x2, 0x0, 0xc, 0x8)
	/app/promql/engine.go:887 +0x1408
github.com/prometheus/prometheus/promql.(*evaluator).eval(0xc0013f1308, 0x2e1c980, 0xc0a9fdd530, 0x2e1c540, 0xc0a9fdd830)
	/app/promql/engine.go:1077 +0x269e
github.com/prometheus/prometheus/promql.(*evaluator).Eval(0xc0013f1308, 0x2e1c980, 0xc0a9fdd530, 0x0, 0x0, 0x0, 0x0)
	/app/promql/engine.go:814 +0x99
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc000128720, 0x2e1c540, 0xc0a9fdd650, 0xc03b91a0c0, 0xc03bd90960, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/app/promql/engine.go:535 +0x5df
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc000128720, 0x2e1c540, 0xc0a9fdd650, 0xc03b91a0c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/app/promql/engine.go:486 +0x4c6
github.com/prometheus/prometheus/promql.(*query).Exec(0xc03b91a0c0, 0x2e1c540, 0xc0a9fdd410, 0xc0163ae200)
	/app/promql/engine.go:188 +0x94
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x2e1c540, 0xc0a9fdd410, 0xc0163ae200, 0x7a, 0xeae97f2, 0xed627a282, 0x40c0d60, 0xc0013f17d0, 0x40ca48, 0x30, ...)
	/app/rules/manager.go:171 +0xc2
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc0014b5f00, 0x2e1c540, 0xc0a9fdd410, 0xeae97f2, 0xed627a282, 0x40c0d60, 0xc0006d04e0, 0xc00072e880, 0x0, 0x2e5bec0, ...)
	/app/rules/recording.go:77 +0xad
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x2e1c540, 0xc001bbe6c0, 0xc0015902a0, 0xeae97f2, 0xed627a282, 0x40c0d60, 0x6, 0x2e5bcc0, 0xc0014b5f00)
	/app/rules/manager.go:572 +0x280
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc0015902a0, 0x2e1c540, 0xc001bbe6c0, 0xeae97f2, 0xed627a282, 0x40c0d60)
	/app/rules/manager.go:637 +0x109
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/app/rules/manager.go:315 +0xc1
github.com/prometheus/prometheus/rules.(*Group).run(0xc0015902a0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:393 +0x4ea
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0015902a0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1033 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc001590380, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc001590380)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1034 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0015901c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0015901c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1035 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc001590540, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc001590540)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1036 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc001590620, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc001590620)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1037 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000f10460, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000f10460)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1030 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc001590000, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc001590000)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1029 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc000357dc0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc000357dc0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 1646 [select]:
net/http.(*persistConn).writeLoop(0xc003b58000)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3965004 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00f94bf40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 171596 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa66e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3100 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143880, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143880)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 9367 [select]:
net/http.(*persistConn).writeLoop(0xc01447c5a0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439272 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc050177340, 0x2de7680, 0xc125af7140, 0xc1fe915bf8, 0xc03deac480, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc050177340, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0018f7ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1fe915ef0, 0x3b9aca00, 0x0, 0xc001690701, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc050177340, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2670, 0xc01a111bc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357873 [select]:
net/http.(*persistConn).writeLoop(0xc0b4182360)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439382 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc050177a20, 0x2de7680, 0xc051c34700, 0xc140c67bf8, 0xc03e05c960, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc050177a20, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01af9cef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc140c67ef0, 0x3b9aca00, 0x0, 0xc023d52f01, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc050177a20, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2d80, 0xc35fd5c6c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439277 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc050177600, 0x2de7680, 0xc00afb7f00, 0xc13b6c7bf8, 0xc032f79800, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc050177600, 0xc029a3b740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc002ef1ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc13b6c7ef0, 0x3b9aca00, 0x0, 0xc001694701, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc050177600, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2800, 0xc352c16c20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357401 [select]:
net/http.(*persistConn).writeLoop(0xc012c36d80)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3433956 [select]:
net/http.(*persistConn).writeLoop(0xc13989d200)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357405 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b63108, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc04c56e398, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc04c56e380, 0xc0a10b6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc04c56e380, 0xc0a10b6000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01dc17db0, 0xc0a10b6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc051952400, 0xc0a10b6000, 0x3823, 0x3823, 0x28, 0x8, 0xc0051808a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc095253758, 0x2dca560, 0xc051952400, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc095253500, 0x7f4ed5b388a0, 0xc00c3cab80, 0x5, 0xc00c3cab80, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc095253500, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc095253500, 0xc0a10a6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0054c1e60, 0xc0a10a6000, 0x1000, 0x1000, 0xc029f52720, 0xc005180c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a109a780)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a109a780, 0x1, 0x0, 0x0, 0x1, 0xc0212e6200, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0054c1e60)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3438779 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0343c5440)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 1357307 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9fd0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001fad918, 0x72, 0x3700, 0x37bc, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001fad900, 0xc008885000, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001fad900, 0xc008885000, 0x37bc, 0x37bc, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f65bb48, 0xc008885000, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc107a715c0, 0xc008885000, 0x37bc, 0x37bc, 0x4, 0x8, 0xc0018f58a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc07f0b1058, 0x2dca560, 0xc107a715c0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc07f0b0e00, 0x7f4ed5b388a0, 0xc01345e9c0, 0x5, 0xc01345e9c0, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc07f0b0e00, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc07f0b0e00, 0xc09a098000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc000fb85a0, 0xc09a098000, 0x1000, 0x1000, 0xc01fece420, 0xc0018f5c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc092b41380)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc092b41380, 0x1, 0x0, 0x0, 0x1, 0xc053f24300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc000fb85a0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3438895 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc017918238, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc017918228)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc017918210, 0xc017d24600, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb4a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001924e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001924e00, 0x3b9aca00, 0x0, 0xc26dd72201, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb4a80, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e000, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 5325 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc016ca39a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439333 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0014b5f80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498140, 0xc02cc00da0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4090037 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0e69fa880, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0e69fa870)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0e69fa868, 0xc0195b7200, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0e69fa840, 0xc0195b7200, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0e69fab00, 0xc0e6337220, 0x1)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0e69fab00, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0e69fab00, 0x223e960, 0xc0e6337220, 0xc001759598, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0e632fd10, 0xc1454ad400, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0e63319f0, 0x0, 0x2de7300, 0xc1fbfdc7c0, 0x21fca4821710c0f0, 0x5e95ab50, 0xc001759658, 0x486d66, 0x5e95ab50)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0e6337200, 0xc00072f280, 0x2e1c500, 0xc0a7d3c240, 0xc0494ee870, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc1fbfdc780)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439610 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0029a2010, 0xc000005b2c)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0029a2000)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc03be7c300, 0x0, 0x0, 0xc0465f7300)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0ddeb4080, 0x2e1c480, 0xc009bf0a80, 0xc2d1d5e5a0, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0ddeb4080, 0x2e1c480, 0xc009bf0a80, 0xc2d1d5e5a0)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 1357715 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b93a0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002579298, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002579280, 0xc0a3900000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002579280, 0xc0a3900000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00663d118, 0xc0a3900000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc0567622c0, 0xc0a3900000, 0x3823, 0x3823, 0x28, 0x8, 0xc01ea638a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a3821058, 0x2dca560, 0xc0567622c0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a3820e00, 0x7f4ed5b388a0, 0xc010764300, 0x5, 0xc010764300, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a3820e00, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a3820e00, 0xc0a38fe000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc002655560, 0xc0a38fe000, 0x1000, 0x1000, 0xc035085b60, 0xc01ea63c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a3889620)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a3889620, 0x1, 0x0, 0x0, 0x1, 0xc00b2ae900, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002655560)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439187 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc283920078)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc283920070)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b4a20, 0x2e1c480, 0xc009bf0b00, 0xc2d1d5e600)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 3245 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143dc0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143dc0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3438973 [sync.Cond.Wait, 148 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049f38238, 0x17)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049f38228)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc049f38210, 0xc001a9d8e0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002515880)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01af9ae00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01af9ae00, 0x3b9aca00, 0x0, 0xc03c2b8001, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002515880, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00381eea0, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 3439168 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0a40, 0xc03c1747d0, 0xc2d1d5e540)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439262 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc045c4cf20, 0x2de7680, 0xc138759b40, 0xc1fe4b9bf8, 0xc03d010120, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc045c4cf20, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001899ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1fe4b9ef0, 0x3b9aca00, 0x0, 0xc0c3e44701, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc045c4cf20, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc1383859e0, 0xc0e0a0bfc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439386 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30d80, 0xc0d8359d00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439374 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc049f38790, 0x2de7680, 0xc04abeb480, 0xc000d6fbf8, 0xc02c122c60, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc049f38790, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc048993ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000d6fef0, 0x3b9aca00, 0x0, 0xc04c1e8701, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc049f38790, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2840944e0, 0xc360366420)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439329 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b740, 0xc0d8359100)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439172 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c310, 0xc021124840)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0d5dc3890, 0xc03b534d40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439556 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0017291e0, 0xc0522e6a80, 0xc021125b00, 0xc03d0308a0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439080 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc013ed05a8, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc013ed0598)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc013ed0580, 0xc023fd5a20, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0507bda00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01af9de00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01af9de00, 0x3b9aca00, 0x0, 0xc26dc0a401, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0507bda00, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003823c20, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 6890 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc019fc0700, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc019fc0700)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439257 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc045c4cdc0, 0x2de7680, 0xc02aa618c0, 0xc0016f1bf8, 0xc032f796e0, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc045c4cdc0, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01a202ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0016f1ef0, 0x3b9aca00, 0x0, 0xc000635f01, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc045c4cdc0, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc138385850, 0xc0e0a0b980)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1816 [select]:
net/http.(*persistConn).writeLoop(0xc002c9a240)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4089111 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc028fc9380, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc028fc9370)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc028fc9368, 0xc00f576400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc028fc9340, 0xc00f576400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc028fc9600, 0xc12eb92c80, 0xc0158c1c01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc028fc9600, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc028fc9600, 0x223e960, 0xc12eb92c80, 0xc047140500, 0x4)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc340253290, 0xc05ad60000, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc028e73720, 0x0, 0x2de7300, 0xc00afb7f40, 0x4313fc, 0xc0158c1e30, 0x45be00, 0xc001869500, 0x4)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc12eb92c60, 0xc0158c1ee8, 0x3, 0x0, 0xc041216ae0, 0xc001853e00, 0xc001853e88)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00afb7f00)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1704 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002dcdae0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357400 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9a20, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc04ddb7b18, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc04ddb7b00, 0xc096c40000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc04ddb7b00, 0xc096c40000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01dc17ca8, 0xc096c40000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc009bb26c0, 0xc096c40000, 0x3823, 0x3823, 0x28, 0x8, 0xc0018658a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc095252958, 0x2dca560, 0xc009bb26c0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc095252700, 0x7f4ed5b388a0, 0xc0015170c0, 0x5, 0xc0015170c0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc095252700, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc095252700, 0xc096c10000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc012c36d80, 0xc096c10000, 0x1000, 0x1000, 0xc0020b7da0, 0xc001865c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0963b5020)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0963b5020, 0x1, 0x0, 0x0, 0x1, 0xc03ac13200, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc012c36d80)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357599 [select]:
net/http.(*persistConn).writeLoop(0xc000fb9c20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1369 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003169a40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439699 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc049f38580, 0xc045b30a80, 0xc01588bb60, 0xc03c45bb60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 2060 [select]:
net/http.(*persistConn).writeLoop(0xc00780a360)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439408 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc001729340, 0xc045b318c0, 0xc010412f60, 0xc03be7d800)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 1368 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002e252c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1327 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faaaa0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1328 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faab40, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1793 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faabe0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1794 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc002faac80, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1370 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc003169ae0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439250 [sync.Cond.Wait, 756 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc050177998, 0x8)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc050177988)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc050177970, 0xc02cc01110, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8359900)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01af9ee00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01af9ee00, 0x3b9aca00, 0x0, 0xc038c11401, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8359900, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc001814240, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 1815 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62678, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0020cdb18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0020cdb00, 0xc00178a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0020cdb00, 0xc00178a000, 0x1000, 0x1000, 0xc00105e280, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc001644020, 0xc00178a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc002c9a240, 0xc00178a000, 0x1000, 0x1000, 0xc001f14720, 0xc000d77c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0019b0480)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0019b0480, 0x1, 0x0, 0x0, 0x1, 0xc023ed4a00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002c9a240)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439294 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30780, 0xc0ddeb4e80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 1357782 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7d88, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d9279198, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d9279180, 0xc0a42e6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d9279180, 0xc0a42e6000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0055a10b0, 0xc0a42e6000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc066ad96e0, 0xc0a42e6000, 0x3823, 0x3823, 0x28, 0x8, 0xc0032148a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a40ce5d8, 0x2dca560, 0xc066ad96e0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a40ce380, 0x7f4ed5b388a0, 0xc01012ee80, 0x5, 0xc01012ee80, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a40ce380, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a40ce380, 0xc0a42e4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc12006a120, 0xc0a42e4000, 0x1000, 0x1000, 0xc0333de8a0, 0xc003214c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a42e0900)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a42e0900, 0x1, 0x0, 0x0, 0x1, 0xc053cc9a00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc12006a120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357538 [select]:
net/http.(*persistConn).writeLoop(0xc0047f8a20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357440 [select]:
net/http.(*persistConn).writeLoop(0xc0047f87e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1645 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62748, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001b1e298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b1e280, 0xc00637b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001b1e280, 0xc00637b000, 0x1000, 0x1000, 0xc00105e500, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc003cec000, 0xc00637b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc003b58000, 0xc00637b000, 0x1000, 0x1000, 0xc003d220c0, 0xc003215c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc000f5a360)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc000f5a360, 0x1, 0x0, 0x0, 0x1, 0xc2d1d5fa00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc003b58000)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357534 [select]:
net/http.(*persistConn).writeLoop(0xc002654fc0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1899 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0054f94a0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1940 [select]:
net/http.(*persistConn).writeLoop(0xc00f65e000)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357482 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b631d8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002914798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002914780, 0xc083133000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002914780, 0xc083133000, 0x1000, 0x1000, 0xc00105edc0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00010a818, 0xc083133000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc012c378c0, 0xc083133000, 0x1000, 0x1000, 0xc01cb32b40, 0xc001897c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc09252f980)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc09252f980, 0x1, 0x0, 0x0, 0x1, 0xc000047b00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc012c378c0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1647 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc00651b040, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357548 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62cf8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc049457d98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc049457d80, 0xc0a0034000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc049457d80, 0xc0a0034000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025ee00, 0xc0a0034000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0010099e0, 0xc0a0034000, 0x1000, 0x1000, 0xc029c40cc0, 0xc001c1ec20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc09635ba40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc09635ba40, 0x1, 0x0, 0x0, 0x1, 0xc2d1d5ff00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0010099e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439165 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc12a3d2458)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc12a3d2450)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b4360, 0x2e1c480, 0xc009bf09c0, 0xc2d1d5e480)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 1357663 [select]:
net/http.(*persistConn).writeLoop(0xc04f204240)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 164979 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0501369a0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0501369a0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439524 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc045c4cf20, 0xc029a3b6e0, 0xc213eb2a80, 0xc03d010120)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 4074549 [IO wait, 4 minutes]:
internal/poll.runtime_pollWait(0x7f4ed4d722b8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0bb1ed098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0bb1ed080, 0xc05b202000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0bb1ed080, 0xc05b202000, 0x1000, 0x1000, 0xc04cdf9a01, 0x10000c04cdf9a80, 0x40c5e40)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0048a0a58, 0xc05b202000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc05abf16d0, 0xc05b202000, 0x1000, 0x1000, 0x72, 0x0, 0xc05ac71a18)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc05abf16c0, 0xc05b202000, 0x1000, 0x1000, 0x10a198c46a402, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc05ac71a10, 0xc05b202000, 0x1000, 0x1000, 0xc04cdf9c38, 0x1e8711a, 0xc05ac719b0)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc05ac6f320)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc05ac6f320, 0x4, 0x10a198c46a402, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc05ac50a00, 0x2e1c480, 0xc052de75c0)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 4090809 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9c90, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002441418, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002441400, 0xc02d476000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002441400, 0xc02d476000, 0x1000, 0x1000, 0xc011f0f900, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc005b49d48, 0xc02d476000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc02a1f70e0, 0xc02d476000, 0x1000, 0x1000, 0xc002c88840, 0xc000d53c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc029c0eea0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc029c0eea0, 0x1, 0x0, 0x0, 0x1, 0xc002c88300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc02a1f70e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3445186 [select]:
net/http.(*persistConn).writeLoop(0xc0036f4b40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357523 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9d60, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc05a592498, 0x72, 0x3600, 0x361d, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc05a592480, 0xc008881800, 0x361d, 0x361d, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc05a592480, 0xc008881800, 0x361d, 0x361d, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00b62f708, 0xc008881800, 0x361d, 0x361d, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc02cbdca20, 0xc008881800, 0x361d, 0x361d, 0x28, 0x8, 0xc002eed8a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc08fbbf758, 0x2dca560, 0xc02cbdca20, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc08fbbf500, 0x7f4ed5b388a0, 0xc00c5bd440, 0x5, 0xc00c5bd440, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc08fbbf500, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc08fbbf500, 0xc095e6e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc00fd68fc0, 0xc095e6e000, 0x1000, 0x1000, 0xc02c07ab40, 0xc002eedc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc09307ad80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc09307ad80, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2d00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00fd68fc0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 4089294 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049afef60, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049afef50)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc049afef48, 0xc02523da00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc049afef20, 0xc02523da00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc049aff1e0, 0xc04ade6720, 0xc0ad109501)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc049aff1e0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc049aff1e0, 0x223e960, 0xc04ade6720, 0xc048a7ec30, 0x70617b)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc048800e70, 0xc0150b5000, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc04d5e20f0, 0x0, 0x2de7300, 0xc000227d80, 0x21eb487101d1fc50, 0x5e95ab23, 0xc048a7ee58, 0x486d66, 0x5e95ab23)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc04ade6700, 0x6, 0xc048a7ef00, 0x441c11, 0xc0d19e7740, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000227d40)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 2059 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7ff8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001b63b98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b63b80, 0xc0099a3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001b63b80, 0xc0099a3000, 0x1000, 0x1000, 0xc00105e500, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009b60a88, 0xc0099a3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00780a360, 0xc0099a3000, 0x1000, 0x1000, 0xc00745bf20, 0xc00321bc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc009bac7e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc009bac7e0, 0x1, 0x0, 0x0, 0x1, 0xc052de4200, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00780a360)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357650 [select]:
net/http.(*persistConn).writeLoop(0xc002654a20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 2212 [select]:
net/http.(*persistConn).writeLoop(0xc00ffea000)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357731 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7708, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d9278a18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d9278a00, 0xc0a418b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d9278a00, 0xc0a418b000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00010b018, 0xc0a418b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc04f193e60, 0xc0a418b000, 0x1000, 0x1000, 0xc13a03c480, 0xc0017c0c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a418cf60)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a418cf60, 0x1, 0x0, 0x0, 0x1, 0xc05346a800, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f193e60)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1941 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9610, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00f64e518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00f64e500, 0xc00f6fb000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00f64e500, 0xc00f6fb000, 0x1000, 0x1000, 0xc00105ea00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00bd5f7e0, 0xc00f6fb000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00f65e120, 0xc00f6fb000, 0x1000, 0x1000, 0xc00f65c4e0, 0xc000bcac20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00f73b9e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00f73b9e0, 0x1, 0x0, 0x0, 0x1, 0xc2f6906300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00f65e120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1985 [select]:
net/http.(*persistConn).writeLoop(0xc00f7717a0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357657 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a8338, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d5d49b98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d5d49b80, 0xc0a39f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d5d49b80, 0xc0a39f8000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009982d28, 0xc0a39f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc04f0890e0, 0xc0a39f8000, 0x1000, 0x1000, 0xc049b3f620, 0xc00310cc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc07d5fb5c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc07d5fb5c0, 0x1, 0x0, 0x0, 0x1, 0xc00b2af800, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f0890e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 2211 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7638, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00f750218, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00f750200, 0xc00fadc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00f750200, 0xc00fadc000, 0x1000, 0x1000, 0xc00105ef00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc010c15e90, 0xc00fadc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00ffea000, 0xc00fadc000, 0x1000, 0x1000, 0xc01067f3e0, 0xc003007c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00fe98000)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00fe98000, 0x1, 0x0, 0x0, 0x1, 0xc2b09f6b00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00ffea000)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439189 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc2839200b8)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc2839200b0)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b4c60, 0x2e1c480, 0xc009bf0b40, 0xc2d1d5e660)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 3439579 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc017918160, 0xc045b30780, 0xc004a2bce0, 0xc03dd2ef60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3438893 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc03be7c300)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 3445128 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71a98, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0b19f9718, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0b19f9700, 0xc2d7d6e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0b19f9700, 0xc2d7d6e000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0083023d8, 0xc2d7d6e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc03b2fc7e0, 0xc2d7d6e000, 0x1000, 0x1000, 0xc2bed4c960, 0xc0086bcc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0232e8660)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0232e8660, 0x1, 0x0, 0x0, 0x1, 0xc034eb1500, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc03b2fc7e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357732 [select]:
net/http.(*persistConn).writeLoop(0xc04f193e60)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3438780 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc050177158, 0x68)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc050177148)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc050177130, 0xc02cc00240, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8358500)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01d531e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01d531e00, 0x3b9aca00, 0x0, 0xc01078d401, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8358500, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0031226c0, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 1357670 [select]:
net/http.(*persistConn).writeLoop(0xc05204cc60)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439283 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c070, 0xc045b30f00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2839200c0, 0xc02c80cc40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357728 [select]:
net/http.(*persistConn).writeLoop(0xc04f08ec60)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357453 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b632a8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00b231e18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00b231e00, 0xc0a2f24000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00b231e00, 0xc0a2f24000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0051da9b0, 0xc0a2f24000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0026546c0, 0xc0a2f24000, 0x1000, 0x1000, 0xc053b9ed20, 0xc001613c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a22d9da0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a22d9da0, 0x1, 0x0, 0x0, 0x1, 0xc003d92900, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0026546c0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357630 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62f68, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d81a6798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d81a6780, 0xc0a3ef0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d81a6780, 0xc0a3ef0000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f65bf78, 0xc0a3ef0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc04f1d3560, 0xc0a3ef0000, 0x1000, 0x1000, 0xc142988e40, 0xc001861c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a3e17ce0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a3e17ce0, 0x1, 0x0, 0x0, 0x1, 0xc00b2afa00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f1d3560)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 2214 [select]:
net/http.(*persistConn).writeLoop(0xc005b64120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357628 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9200, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002931898, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002931880, 0xc0a39dc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002931880, 0xc0a39dc000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f65bee0, 0xc0a39dc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc04f08e7e0, 0xc0a39dc000, 0x1000, 0x1000, 0xc1429884e0, 0xc00517ec20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a39ba4e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a39ba4e0, 0x1, 0x0, 0x0, 0x1, 0xc2f6907e00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f08e7e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 5521 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f090, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002914a18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002914a00, 0xc00dbd2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002914a00, 0xc00dbd2000, 0x1000, 0x1000, 0xc0044d4280, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0073c9b78, 0xc00dbd2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00fd58a20, 0xc00dbd2000, 0x1000, 0x1000, 0xc01367a8a0, 0xc01543bc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc000ed7140)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc000ed7140, 0x1, 0x0, 0x0, 0x1, 0xc00b2aea00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00fd58a20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357649 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62b58, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00b231f98, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00b231f80, 0xc0a3364000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00b231f80, 0xc0a3364000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00010aa00, 0xc0a3364000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc11ffd6a40, 0xc0a3364000, 0x3823, 0x3823, 0x28, 0x8, 0xc0032188a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a3336258, 0x2dca560, 0xc11ffd6a40, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a3336000, 0x7f4ed5b388a0, 0xc00b3616c0, 0x5, 0xc00b3616c0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a3336000, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a3336000, 0xc0a096d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc002654a20, 0xc0a096d000, 0x1000, 0x1000, 0xc053b9ef60, 0xc003218c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a2afba40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a2afba40, 0x1, 0x0, 0x0, 0x1, 0xc032366400, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002654a20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439255 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a7490, 0xc3548d1020)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc138385810, 0xc0e0a0b0e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439293 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c0e0, 0xc045b31200)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2839203d0, 0xc02c80d4e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357775 [select]:
net/http.(*persistConn).writeLoop(0xc0b41607e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 2213 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7568, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc005b3c518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc005b3c500, 0xc012217000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc005b3c500, 0xc012217000, 0x1000, 0x1000, 0xc002963e00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc010c15ed0, 0xc012217000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc005b64120, 0xc012217000, 0x1000, 0x1000, 0xc01067f500, 0xc0086c0c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00fe981e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00fe981e0, 0x1, 0x0, 0x0, 0x1, 0xc0135d5c00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc005b64120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 4090963 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc03ab1cf60, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc03ab1cf50)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc03ab1cf48, 0xc05a9c4a00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc03ab1cf20, 0xc05a9c4a00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc03ab1d1e0, 0xc0e19d2820, 0x2a5fa01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc03ab1d1e0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc03ab1d1e0, 0x223e960, 0xc0e19d2820, 0x203002, 0x203002)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc075806e70, 0xc055955c00, 0x400, 0x400, 0x40, 0x40, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0399fdcc0, 0x0, 0x2de7300, 0xc00f6e4180, 0x219e41c301cb31a0, 0x5e95ab91, 0xc002203658, 0x486d66, 0x5e95ab91)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0e19d2800, 0x70500d, 0x70242c, 0xc011f0f900, 0x2e1c500, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00f6e4140)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3440243 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ece0d3250, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002097798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002097780, 0xc04746d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002097780, 0xc04746d000, 0x1000, 0x1000, 0xc010797e00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004fc1b80, 0xc04746d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc07a200d80, 0xc04746d000, 0x1000, 0x1000, 0xc2b5e79d40, 0xc004d5dc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0474696e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0474696e0, 0x1, 0x0, 0x0, 0x1, 0xc023c96e00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc07a200d80)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1994 [select]:
net/http.(*persistConn).writeLoop(0xc00db04120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1993 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7be8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0021bab18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0021bab00, 0xc00e4a6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0021bab00, 0xc00e4a6000, 0x1000, 0x1000, 0xc00105e280, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f956398, 0xc00e4a6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00db04120, 0xc00e4a6000, 0x1000, 0x1000, 0xc00e45c420, 0xc0018f1c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00c6d7aa0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00c6d7aa0, 0x1, 0x0, 0x0, 0x1, 0xc005a17500, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00db04120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357836 [select]:
net/http.(*persistConn).writeLoop(0xc0b41939e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439086 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c4f0000, 0xc2dde426c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc1340b03a0, 0xc04f397900)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4075705 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed4d4eef0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001b36818, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b36800, 0xc0197a6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001b36800, 0xc0197a6000, 0x1000, 0x1000, 0xc20115ba01, 0x10000c20115ba80, 0x40c5fc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0052a4360, 0xc0197a6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc014d6a710, 0xc0197a6000, 0x1000, 0x1000, 0x72, 0x0, 0xc0a4544c38)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc014d6a700, 0xc0197a6000, 0x1000, 0x1000, 0x10a2a14046572, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc0a4544c30, 0xc0197a6000, 0x1000, 0x1000, 0xc20115bc38, 0x1e8711a, 0xc0a4544bd0)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc014235f20)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc014235f20, 0x4, 0x10a2a14046572, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc018e1a460, 0x2e1c480, 0xc2024abb40)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 4090880 [select]:
net/http.(*persistConn).writeLoop(0xc0025f4a20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439087 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b318c0, 0xc0507bda00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439071 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc03be22420)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 5392 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f160, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002894118, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002894100, 0xc011a4c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002894100, 0xc011a4c000, 0x1000, 0x1000, 0xc0044d43c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc013805118, 0xc011a4c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00c0acea0, 0xc011a4c000, 0x1000, 0x1000, 0xc01367a480, 0xc00160ec20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0079a8d80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0079a8d80, 0x1, 0x0, 0x0, 0x1, 0xc2f6906100, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00c0acea0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439220 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc001728f48, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc001728f38)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001728f20, 0xc001a97ec0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc126a7e780)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01e82be00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01e82be00, 0x3b9aca00, 0x0, 0xc26dd94001, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc126a7e780, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122870, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3439340 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a20, 0xc0d8359500)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439240 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0ddeb4300, 0x2e1c480, 0xc009bf0b00, 0xc2d1d5e600)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc283920070, 0x2e1c480, 0xc009bf0b00, 0xc2d1d5e600, 0x2dcba20, 0xc0ddeb4300)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 4090590 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a77d8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0020eb118, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0020eb100, 0xc041717000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0020eb100, 0xc041717000, 0x1000, 0x1000, 0xc011f0f900, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0071e37a8, 0xc041717000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc008468900, 0xc041717000, 0x1000, 0x1000, 0xc054f23860, 0xc00185fc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0416e75c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0416e75c0, 0x1, 0x0, 0x0, 0x1, 0xc054f23700, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc008468900)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 2257 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f4a0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc013344b18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc013344b00, 0xc0143ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc013344b00, 0xc0143ca000, 0x1000, 0x1000, 0xc00105ea00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc012de4288, 0xc0143ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0143c8000, 0xc0143ca000, 0x1000, 0x1000, 0xc013fae1e0, 0xc0018f6c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0139ace40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0139ace40, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2b00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0143c8000)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 2258 [select]:
net/http.(*persistConn).writeLoop(0xc0143c8000)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439378 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc05830f300)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a7640, 0xc02cc010f0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439367 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc049f38580, 0x2de7680, 0xc00049d280, 0xc13b60fbf8, 0xc03c45bb60, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc049f38580, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000bc5ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc13b60fef0, 0x3b9aca00, 0x0, 0xc002201f01, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc049f38580, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc284094360, 0xc360366140)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 171593 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa6500, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 5323 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0161c3e00, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3444494 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7f28, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0d2896298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0d2896280, 0xc0c998d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0d2896280, 0xc0c998d000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009b607e0, 0xc0c998d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0d285c120, 0xc0c998d000, 0x1000, 0x1000, 0xc0135d4360, 0xc01e828c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0c7d37f80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0c7d37f80, 0x1, 0x0, 0x0, 0x1, 0xc2b5379000, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0d285c120)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3086461 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71c38, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc125af2418, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc125af2400, 0xc1244c8000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc125af2400, 0xc1244c8000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00f65bd98, 0xc1244c8000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc139997ea0, 0xc1244c8000, 0x3823, 0x3823, 0x28, 0x8, 0xc01e82f8a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0b48fa5d8, 0x2dca560, 0xc139997ea0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0b48fa380, 0x7f4ed5b388a0, 0xc05a0afec0, 0x5, 0xc05a0afec0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0b48fa380, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0b48fa380, 0xc12c072000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc1415a37a0, 0xc12c072000, 0x1000, 0x1000, 0xc054608c60, 0xc01e82fc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0ac055380)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0ac055380, 0x1, 0x0, 0x0, 0x1, 0xc00bc94000, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc1415a37a0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439398 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0d8358200)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498220, 0xc017d251e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4086934 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0ac36c720, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0ac36c710)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0ac36c708, 0xc053821600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0ac36c6e0, 0xc053821600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0ac36c9a0, 0xc012ea8560, 0xc001b78a01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0ac36c9a0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0ac36c9a0, 0x223e960, 0xc012ea8560, 0xc019ee5900, 0xc034ddba60)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc018304c30, 0xc0c8c30c00, 0x400, 0x400, 0xc033ecbb58, 0xc0e7d66a20, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0d21fbae0, 0x0, 0x2de7300, 0xc0c08ca580, 0x21d63bc800000000, 0x5e95aa83, 0xc001a07e58, 0x486d66, 0x5e95aa83)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc012ea8540, 0xffff000000000008, 0xc001a07e50, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0c08ca540)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3101 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143960, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143960)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439357 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0179186e0, 0x2de7680, 0xc008414940, 0xc200e91bf8, 0xc03c489e60, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0179186e0, 0xc3548d0a20, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc059426ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc200e91ef0, 0x3b9aca00, 0x0, 0xc00ed18701, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0179186e0, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920710, 0xc02c80dee0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3438971 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc001728868, 0x6e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc001728858)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001728840, 0xc001a970f0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002947680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001874e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001874e00, 0x3b9aca00, 0x0, 0xc0530b8f01, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002947680, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00381ee10, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 3439216 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c150, 0xc01588a120)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2840941a0, 0xc02bd83540)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 176445 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc049c635e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc049c635e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 171591 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa63c0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439689 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc049f382c0, 0xc3548d0a80, 0xc036274b40, 0xc03dd2fda0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439218 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0343c5500)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 3439160 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0900, 0xc03be92a50, 0xc2d1d5e360)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439222 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0d8358100, 0x2e1c480, 0xc009bf09c0, 0xc2d1d5e480)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc12a3d2450, 0x2e1c480, 0xc009bf09c0, 0xc2d1d5e480, 0x2dcba20, 0xc0d8358100)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3439332 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x38, 0xc2024ab000, 0x40)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc01a211dd8, 0xc000391df0, 0x40ca48)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000391f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01a211f40, 0xdf8475800, 0x0, 0x42eb01, 0xc0522e7440)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0014b5f80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498140, 0xc02cc00d90)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439082 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0507bd880, 0x2e1c480, 0xc0100d0400, 0xc2d1d5f680)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc1340b0390, 0x2e1c480, 0xc0100d0400, 0xc2d1d5f680, 0x2dcba20, 0xc0507bd880)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3439236 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc03be7c3c0)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 5350 [select]:
net/http.(*persistConn).writeLoop(0xc00f863e60)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4090968 [select]:
net/http.(*persistConn).writeLoop(0xc002c5ed80)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439207 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4b22a0, 0xc053f24ae0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc284094000, 0xc02bd82f60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3438782 [sync.Cond.Wait, 778 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc001728bd8, 0x1)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc001728bc8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001728bb0, 0xc001a97850, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002947e80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01543ae00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01543ae00, 0x3b9aca00, 0x0, 0xc03c2e4001, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002947e80, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122750, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 186532 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04a57b0e0, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3965329 [select]:
net/http.(*persistConn).writeLoop(0xc0030cea20)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439609 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc017918420, 0xc045b30780, 0xc02c230720, 0xc03e23f980)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 4090055 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc10a9fa9e0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc10a9fa9d0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc10a9fa9c8, 0xc00bdf6000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc10a9fa9a0, 0xc00bdf6000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc10a9fac60, 0xc107d1bc20, 0xc001672301)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc10a9fac60, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc10a9fac60, 0x223e960, 0xc107d1bc20, 0xc00170a430, 0x70617b)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc107d2b470, 0xc0ddec2400, 0x400, 0x400, 0x0, 0xc190a66000, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc107cf8e10, 0x0, 0x2de7300, 0xc10a9f85c0, 0x21cdd61f0009ad80, 0x5e95ab55, 0xc00170a658, 0x486d66, 0x5e95ab55)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc107d1bc00, 0x0, 0xc010da71f8, 0x4590b0, 0xc010da71a0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc10a9f8580)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439362 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc049f38420, 0x2de7680, 0xc0087648c0, 0xc200e95bf8, 0xc03be7cc60, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc049f38420, 0xc045b30a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000bc8ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc200e95ef0, 0x3b9aca00, 0x0, 0xc00181df01, 0xc045b30a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc049f38420, 0xc045b30a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2840941e0, 0xc02bd836c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4071962 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ece0d3b40, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00219ad18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00219ad00, 0xc000ee1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00219ad00, 0xc000ee1000, 0x1000, 0x1000, 0xc200ff9a01, 0x10000c200ff9a80, 0x40c5fc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004be4340, 0xc000ee1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc005c4e080, 0xc000ee1000, 0x1000, 0x1000, 0x72, 0x0, 0xc05769bf28)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc005c4e070, 0xc000ee1000, 0x1000, 0x1000, 0x10a2aa613f8dc, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc05769bf20, 0xc000ee1000, 0x1000, 0x1000, 0xc200ff9c38, 0x1e8711a, 0xc05769bec0)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc01a5128a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc01a5128a0, 0x4, 0x10a2aa613f8dc, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc005bee140, 0x2e1c480, 0xc124f68f40)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 3439188 [select, 6 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0b00, 0xc03c242690, 0xc2d1d5e600)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439233 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0ddeb4080, 0x2e1c480, 0xc009bf0a80, 0xc2d1d5e5a0)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc283920030, 0x2e1c480, 0xc009bf0a80, 0xc2d1d5e5a0, 0x2dcba20, 0xc0ddeb4080)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3425687 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7498, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc2ed309c98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc2ed309c80, 0xc2f4cbe000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc2ed309c80, 0xc2f4cbe000, 0x1000, 0x1000, 0xc010787540, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004be5c90, 0xc2f4cbe000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc2efefbd40, 0xc2f4cbe000, 0x1000, 0x1000, 0xc2ed2c3260, 0xc01a212c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc2eed79620)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc2eed79620, 0x1, 0x0, 0x0, 0x1, 0xc053f35900, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc2efefbd40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439284 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30780, 0xc0ddeb4680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439158 [select, 8 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf08c0, 0xc03bc19ae0, 0xc2d1d5e300)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439226 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0501772b8, 0x1693e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0501772a8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc050177290, 0xc02cc00510, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8358900)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0018f4e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0018f4e00, 0x3b9aca00, 0x0, 0xc01078d701, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8358900, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122b40, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 4090810 [select]:
net/http.(*persistConn).writeLoop(0xc02a1f70e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439361 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30a80, 0xc002515c80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3444657 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ece0d33f0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0c3b1fa18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0c3b1fa00, 0xc0c518a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0c3b1fa00, 0xc0c518a000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc02425de48, 0xc0c518a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0c474a480, 0xc0c518a000, 0x1000, 0x1000, 0xc05a406660, 0xc05fa5dc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0c412e480)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0c412e480, 0x1, 0x0, 0x0, 0x1, 0xc0212e7800, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0c474a480)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439164 [select, 8 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0980, 0xc03c0b4910, 0xc2d1d5e420)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3177937 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc002b9e000, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc002b9e000)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439380 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a7650, 0xc0522e77a0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2d40, 0xc35fd5c600)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 186581 [select]:
net/http.(*persistConn).writeLoop(0xc054104900)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4089512 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc06617d0c0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc06617d0b0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc06617d0a8, 0xc0120cb800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc06617d080, 0xc0120cb800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc06617d340, 0xc066324e80, 0xc00f7e7501)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc06617d340, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc06617d340, 0x223e960, 0xc066324e80, 0x203002, 0x203002)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0650f7bf0, 0xc00fb8bc00, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc066328d70, 0x0, 0x2de7300, 0xc008dd4c00, 0x219ed29d01d1efc0, 0x5e95ab2d, 0xc0caa4c658, 0x486d66, 0x5e95ab2d)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc066324e60, 0x8, 0xc0caa4c650, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc008dd4bc0)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 4087595 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0568aa880, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0568aa870)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0568aa868, 0xc125d36a00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0568aa840, 0xc125d36a00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0568aab00, 0xc053588e20, 0x1)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0568aab00, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0568aab00, 0x223e960, 0xc053588e20, 0xc0158bd598, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc053581a40, 0xc04c186c00, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05355cd20, 0x0, 0x2de7300, 0xc008414980, 0x2226336d01d66330, 0x5e95aab0, 0xc0158bd658, 0x486d66, 0x5e95aab0)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc053588e00, 0x8, 0xc0158bd650, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc008414940)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439166 [select, 8 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf09c0, 0xc03c0b5860, 0xc2d1d5e480)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439256 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b6e0, 0xc05830f400)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3965213 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4e940, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0020efa18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0020efa00, 0xc0d28f9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0020efa00, 0xc0d28f9000, 0x1000, 0x1000, 0xc14bfdfa01, 0x10000c14bfdfa80, 0x40c5e40)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004d2f010, 0xc0d28f9000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc038752010, 0xc0d28f9000, 0x1000, 0x1000, 0x72, 0x0, 0xc04979e5d8)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc038752000, 0xc0d28f9000, 0x1000, 0x1000, 0x10a44d3d170f7, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc04979e5d0, 0xc0d28f9000, 0x1000, 0x1000, 0xc14bfdfc38, 0x1e8711a, 0xc04979e570)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc03f173800)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc03f173800, 0x4, 0x10a44d3d170f7, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc0378e8b40, 0x2e1c480, 0xc0088e6780)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 3439402 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c380, 0xc045b31920)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920850, 0xc352eb81c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439186 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0a80, 0xc03c175720, 0xc2d1d5e5a0)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3086486 [select]:
net/http.(*persistConn).writeLoop(0xc12c684fc0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357439 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9e30, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc05bbc2c18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc05bbc2c00, 0xc09342e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc05bbc2c00, 0xc09342e000, 0x1000, 0x1000, 0xc00105edc0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc012fb35c0, 0xc09342e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0047f87e0, 0xc09342e000, 0x1000, 0x1000, 0xc04caabb60, 0xc0018f3c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc057c11980)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc057c11980, 0x1, 0x0, 0x0, 0x1, 0xc2b09f6f00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0047f87e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439302 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0522e6a80, 0xc126a7e280)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439369 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc051c48c80, 0x2000, 0x6)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000bc8dd8, 0xc001735618, 0x4e3f56)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001735740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000bc8f40, 0xdf8475800, 0x0, 0x42eb01, 0xc01588a420)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4400)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c1b0, 0xc0021f6350)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3103 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143a40, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143a40)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3246 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143b20, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143b20)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3104 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb81340, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb81340)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3440112 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc11cb76620, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc11cb76620)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3409 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc010143c00, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc010143c00)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3248 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb80fc0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb80fc0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3394 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb81500, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb81500)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3440089 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc11cb768c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc11cb768c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3395 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb81420, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb81420)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4091076 [select]:
net/http.(*persistConn).writeLoop(0xc039d39b00)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439212 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc002515100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4b2220, 0xc001a9dbb0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 9375 [select]:
net/http.(*persistConn).writeLoop(0xc01ad7f680)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 9374 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4e870, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002163218, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002163200, 0xc01490f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002163200, 0xc01490f000, 0x1000, 0x1000, 0xc001418a00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004fc0110, 0xc01490f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc01ad7f680, 0xc01490f000, 0x1000, 0x1000, 0xc0059567e0, 0xc003005c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc015484540)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc015484540, 0x1, 0x0, 0x0, 0x1, 0xc2c0f57500, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc01ad7f680)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3444608 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed4d4e600, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0c52c4218, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0c52c4200, 0xc00dbd5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0c52c4200, 0xc00dbd5000, 0x1000, 0x1000, 0xc0043f88c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00bd5e2e0, 0xc00dbd5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0c3b23440, 0xc00dbd5000, 0x1000, 0x1000, 0xc01fecfda0, 0xc004d5ec20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0c432a5a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0c432a5a0, 0x1, 0x0, 0x0, 0x1, 0xc053d36400, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0c3b23440)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439204 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc049f38160, 0x2de7680, 0xc10a9f8580, 0xc1f8621bf8, 0xc03be22e40, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc049f38160, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00212eef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc1f8621ef0, 0x3b9aca00, 0x0, 0xc001732701, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc049f38160, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc13a3cdda0, 0xc02bd82760)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439373 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b318c0, 0xc0e1598580)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3964169 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4e7a0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0b6af3f18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0b6af3f00, 0xc0066e7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0b6af3f00, 0xc0066e7000, 0x1000, 0x1000, 0xc0a3cdfa01, 0x10000c0a3cdfa80, 0x40c5f40)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0154c7c88, 0xc0066e7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc070e77350, 0xc0066e7000, 0x1000, 0x1000, 0x72, 0x0, 0xc07acc4878)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc070e77340, 0xc0066e7000, 0x1000, 0x1000, 0x10a402c2ac6c3, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc07acc4870, 0xc0066e7000, 0x1000, 0x1000, 0xc0a3cdfc38, 0x1e8711a, 0xc07acc4810)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc032f78180)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc032f78180, 0x4, 0x10a402c2ac6c3, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc078ea9cc0, 0x2e1c480, 0xc007935940)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 3439159 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc1383857c8)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc1383857c0)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0036cc5a0, 0x2e1c480, 0xc009bf0900, 0xc2d1d5e360)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 3443995 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2ea00, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3445129 [select]:
net/http.(*persistConn).writeLoop(0xc03b2fc7e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3086462 [select]:
net/http.(*persistConn).writeLoop(0xc1415a37a0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357536 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a8198, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc126a7e418, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc126a7e400, 0xc0a9a42000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc126a7e400, 0xc0a9a42000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00bd5e088, 0xc0a9a42000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc063661ac0, 0xc0a9a42000, 0x3823, 0x3823, 0x28, 0x8, 0xc0017188a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a9a1a258, 0x2dca560, 0xc063661ac0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a9a1a000, 0x7f4ed5b388a0, 0xc008dd43c0, 0x5, 0xc008dd43c0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a9a1a000, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a9a1a000, 0xc0a9a38000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0b4182360, 0xc0a9a38000, 0x1000, 0x1000, 0xc03453d0e0, 0xc001718c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a9a1ca80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a9a1ca80, 0x1, 0x0, 0x0, 0x1, 0xc0212f1000, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b4182360)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3177680 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc01943bea0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc01943bea0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 171595 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc047fa6640, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439261 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b6e0, 0xc05830f980)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439339 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a75e0, 0xc0522e7560)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2bb0, 0xc35fd5c320)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357835 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d714e8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc136c76518, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc136c76500, 0xc0b1294000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc136c76500, 0xc0b1294000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0016dbf20, 0xc0b1294000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc05442ee00, 0xc0b1294000, 0x3823, 0x3823, 0x28, 0x8, 0xc0017c48a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a4d24cd8, 0x2dca560, 0xc05442ee00, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a4d24a80, 0x7f4ed5b388a0, 0xc0416087c0, 0x5, 0xc0416087c0, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a4d24a80, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a4d24a80, 0xc0b11b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0b41939e0, 0xc0b11b4000, 0x1000, 0x1000, 0xc124e20120, 0xc0017c4c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0b11b2060)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0b11b2060, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2a00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b41939e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357308 [select]:
net/http.(*persistConn).writeLoop(0xc000fb85a0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439211 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc0017366dc, 0x2, 0x2)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00212edd8, 0x0, 0xc145406ca0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001736740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00212ef40, 0xdf8475800, 0x0, 0x42eb01, 0xc01588a0c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc002515100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4b2220, 0xc001a9dba0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4083011 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0501771e0, 0xc029a3bb00, 0xc2cb7f4480, 0xc0255241e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439315 [select, 4 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc045c4d080, 0x2de7680, 0xc143042840, 0xc05740fbf8, 0xc03deadb60, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc045c4d080, 0xc029a3b6e0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01d52cef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc05740fef0, 0x3b9aca00, 0x0, 0xc001a03701, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc045c4d080, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc138385b60, 0xc0541bc3c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357776 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7b18, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc126d42618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc126d42600, 0xc0aa0e4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc126d42600, 0xc0aa0e4000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025f9a8, 0xc0aa0e4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0b4160b40, 0xc0aa0e4000, 0x1000, 0x1000, 0xc03453dce0, 0xc002ef0c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a9d02360)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a9d02360, 0x1, 0x0, 0x0, 0x1, 0xc2ae0aa000, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b4160b40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439208 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a80, 0xc002515880)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 1357528 [select]:
net/http.(*persistConn).writeLoop(0xc0054c19e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4083013 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0291ad640, 0xc000000000)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0291ad630)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0291ad628, 0xc007ef5e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0291ad600, 0xc007ef5e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0291ade40, 0xc042092820, 0xc011dfca01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0291ade40, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0291ade40, 0x223e960, 0xc042092820, 0x203002, 0x203002)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0425e8960, 0xc0bde04c00, 0x400, 0x400, 0x40, 0x40, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc047cfc1e0, 0x0, 0x2de7300, 0xc0f47e8380, 0xc001cb3ad0, 0x2a5fc88, 0x40a2bf0, 0xc021d14e50, 0xc021d14f50)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc042092800, 0x6, 0xc021d14f00, 0x441c11, 0xc02fdae120, 0x6, 0xc02fdae120)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0f47e8340)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3438781 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0017289c8, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0017289b8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0017289a0, 0xc001a974c0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002947a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00281de00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00281de00, 0x3b9aca00, 0x0, 0xc05552fd01, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002947a80, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0031225a0, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 1357438 [select]:
net/http.(*persistConn).writeLoop(0xc0076027e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3440236 [select]:
net/http.(*persistConn).writeLoop(0xc0ba629200)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439701 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc002c685d0, 0xc1000000b4)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc002c685c0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc0343c5500, 0x0, 0x0, 0xc0d22aa500)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0d8358100, 0x2e1c480, 0xc009bf09c0, 0xc2d1d5e480, 0xc001b97f00)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0d8358100, 0x2e1c480, 0xc009bf09c0, 0xc2d1d5e480)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3439400 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0d8358180)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498290, 0xc017d25200)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439065 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc045c4cd38, 0x10efe)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc045c4cd28)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc045c4cd10, 0xc0019d9dd0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc05830f400)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01a208e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01a208e00, 0x3b9aca00, 0x0, 0xc0047d0301, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc05830f400, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc001814000, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 1357639 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9880, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc048343c98, 0x72, 0x3600, 0x361d, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc048343c80, 0xc00f3e4000, 0x361d, 0x361d, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc048343c80, 0xc00f3e4000, 0x361d, 0x361d, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00663d0a0, 0xc00f3e4000, 0x361d, 0x361d, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc134638700, 0xc00f3e4000, 0x361d, 0x361d, 0x28, 0x8, 0xc0018928a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc09a152cd8, 0x2dca560, 0xc134638700, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc09a152a80, 0x7f4ed5b388a0, 0xc05a4ed800, 0x5, 0xc05a4ed800, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc09a152a80, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc09a152a80, 0xc0a389c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc00c27eb40, 0xc0a389c000, 0x1000, 0x1000, 0xc053b9f260, 0xc001892c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a2ef94a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a2ef94a0, 0x1, 0x0, 0x0, 0x1, 0xc02fbbb600, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00c27eb40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439088 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc013ed0630, 0x2de7680, 0xc00f6e4140, 0xc201e4bbf8, 0xc03d0102a0, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc013ed0630, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc01d52fef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc201e4bef0, 0x3b9aca00, 0x0, 0xc001b24701, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc013ed0630, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc1340b03e0, 0xc04f3979c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 9366 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4ea10, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001d88098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001d88080, 0xc01a2c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001d88080, 0xc01a2c2000, 0x1000, 0x1000, 0xc001418640, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01d57aaa0, 0xc01a2c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc01447c5a0, 0xc01a2c2000, 0x1000, 0x1000, 0xc01d026960, 0xc01afa0c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc019b5ad20)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc019b5ad20, 0x1, 0x0, 0x0, 0x1, 0xc052de4f00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc01447c5a0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 4090642 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc015966300, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0159662f0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0159662e8, 0xc088eb2000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0159662c0, 0xc088eb2000, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc015966580, 0xc073305ce0, 0xc0021fe401)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc015966580, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc015966580, 0x223e960, 0xc073305ce0, 0xc03f7303c0, 0x4)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc047a82ff0, 0xc0a409a400, 0x400, 0x400, 0x0, 0x2, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc01393f090, 0x0, 0x2de7300, 0xc138759b80, 0x21bab922040de4e0, 0x5e95ab73, 0xc0021fe658, 0x486d66, 0x5e95ab73)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc073305cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc138759b40)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439550 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0179182c0, 0xc045b30780, 0xc01588b8c0, 0xc03c45b800)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439213 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x1f4)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0086bedd8, 0x0, 0xc04c1eedf8)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc04c1eef40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0086bef40, 0xdf8475800, 0x0, 0x42eb01, 0xc01588a3c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c0d0, 0xc001a9dbe0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 162650 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0383d5c20, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1358010 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4eae0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc126d43b18, 0x72, 0xdb00, 0xdb6b, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc126d43b00, 0xc0babb0000, 0xdb6b, 0xdb6b, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc126d43b00, 0xc0babb0000, 0xdb6b, 0xdb6b, 0x203011, 0x0, 0xd8f0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025fac0, 0xc0babb0000, 0xdb6b, 0xdb6b, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc0469970e0, 0xc0babb0000, 0xdb6b, 0xdb6b, 0x4dc, 0x2dca760, 0xc0641b8970)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a526fe58, 0x2dca560, 0xc0469970e0, 0x40a1b5, 0x225d900, 0x257de00)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a526fc00, 0x2dccd80, 0xc01025fac0, 0x5, 0xc01025fac0, 0x251)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a526fc00, 0x0, 0x0, 0xc04514f840)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a526fc00, 0xc0b1b2f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc0b1aec720, 0xc0a41c50d8, 0x9, 0x9, 0x3fc4, 0x248, 0x0)
	/usr/local/go/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x2dca380, 0xc0b1aec720, 0xc0a41c50d8, 0x9, 0x9, 0x9, 0x0, 0x0, 0xc0641b8f78)
	/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0a41c50d8, 0x9, 0x9, 0x2dca380, 0xc0b1aec720, 0x0, 0x0, 0xc04699b1a0, 0x0)
	/app/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0a41c50a0, 0xc04699b1a0, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0641b8fb8, 0x0, 0xc0a38dc000)
	/app/vendor/golang.org/x/net/http2/transport.go:1746 +0x8c
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0b1ac2180)
	/app/vendor/golang.org/x/net/http2/transport.go:1674 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
	/app/vendor/golang.org/x/net/http2/transport.go:674 +0x62f

goroutine 3439228 [sync.Cond.Wait, 10 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0501776d8, 0x2d8)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0501776c8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0501776b0, 0xc02cc00ad0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8359100)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc005182e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc005182e00, 0x3b9aca00, 0x0, 0xc03c2e4801, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8359100, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122bd0, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 3439306 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4981c0, 0xc03e31ac60)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e09f0, 0xc0b1e54f20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439385 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c2a0, 0xc0522e78c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2ec0, 0xc35fd5c9a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 6730 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc019fc0540, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc019fc0540)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4082750 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d719c8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc133b9e718, 0x72, 0x3800, 0x3823, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc133b9e700, 0xc1403ce000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc133b9e700, 0xc1403ce000, 0x3823, 0x3823, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004be4fc0, 0xc1403ce000, 0x3823, 0x3823, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc063b5bee0, 0xc1403ce000, 0x3823, 0x3823, 0x28, 0x8, 0xc00517f8a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc1319ed758, 0x2dca560, 0xc063b5bee0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc1319ed500, 0x7f4ed5b388a0, 0xc1228f0540, 0x5, 0xc1228f0540, 0x12)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc1319ed500, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc1319ed500, 0xc140328000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc12e07fd40, 0xc140328000, 0x1000, 0x1000, 0xc2b1bbb440, 0xc00517fc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc14031a000)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc14031a000, 0x1, 0x0, 0x0, 0x1, 0xc2b09f6c00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc12e07fd40)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3440090 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00fea3910, 0xc0000000b7)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00fea3900)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc03be22420, 0x0, 0x0, 0xc041edbe00)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc05830f280, 0x2e1c480, 0xc009bf0940, 0xc2d1d5e3c0, 0xc11cb76800)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc05830f280, 0x2e1c480, 0xc009bf0940, 0xc2d1d5e3c0)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3439285 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc017918160, 0x2de7680, 0xc13ea3b980, 0xc020e0dbf8, 0xc03dd2ef60, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc017918160, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0017c7ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc020e0def0, 0x3b9aca00, 0x0, 0xc0004e1701, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc017918160, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920100, 0xc02c80cd60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439176 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x40618a, 0xc045b30d80, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc003fb1dd8, 0x0, 0xc02e3e3e28)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc02e3e3f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003fb1f40, 0xdf8475800, 0x0, 0x42eb01, 0xc021124c60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4580)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c370, 0xc002606400)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439353 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4380)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c220, 0xc017d24e90)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3440235 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ece0d3180, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc125af2a98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc125af2a80, 0xc042a23000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc125af2a80, 0xc042a23000, 0x1000, 0x1000, 0xc010797e00, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004d2f498, 0xc042a23000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0ba629200, 0xc042a23000, 0x1000, 0x1000, 0xc03b1517a0, 0xc00183cc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc042b494a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc042b494a0, 0x1, 0x0, 0x0, 0x1, 0xc2d1d5fd00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0ba629200)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 1357774 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a78a8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc126d42298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc126d42280, 0xc0aa0e2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc126d42280, 0xc0aa0e2000, 0x1000, 0x1000, 0xc00105e3c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025f9a0, 0xc0aa0e2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0b41607e0, 0xc0aa0e2000, 0x1000, 0x1000, 0xc0333deea0, 0xc0086c1c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a9d022a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a9d022a0, 0x1, 0x0, 0x0, 0x1, 0xc2b09f6900, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b41607e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439246 [sync.Cond.Wait, 778 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0179187b8, 0x1)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0179187a8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc017918790, 0xc017d25220, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb5a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc003004e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003004e00, 0x3b9aca00, 0x0, 0xc03c30e801, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb5a80, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e480, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 4086600 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc1404a2040, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc1404a2030)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc1404a2028, 0xc00142d800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc1404a2000, 0xc00142d800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc1404a22c0, 0xc13ea477e0, 0x2a5faa8)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc1404a22c0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc1404a22c0, 0x223e960, 0xc13ea477e0, 0xc001759d98, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc1402be480, 0xc0295fb000, 0x400, 0x400, 0x0, 0xc0015a9180, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc14049e0f0, 0x0, 0x2de7300, 0xc13ea3b140, 0x40de4e0, 0xc001759e70, 0x1e8826a, 0xc0016dbf00, 0xc056bab961)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc13ea477c0, 0x0, 0x2dccd20, 0x40906b, 0xc00162e000, 0x4313fc, 0xc001759ed0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc13ea3b100)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 186533 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc04a57b180, 0x6fc23ac00, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439120 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b498150, 0xc03e31a9c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e06e0, 0xc04a455fc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439227 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc050177418, 0x52)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc050177408)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0501773f0, 0xc02cc00800, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8358d00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc022b87e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc022b87e00, 0x3b9aca00, 0x0, 0xc26dd94401, 0xc029a3b740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8358d00, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122a20, 0xc029a3b740)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3439181 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0507bd900)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c4f00d0, 0xc002606450)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439108 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4b2230, 0xc03e31a300)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e03f0, 0xc04a455980)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3424379 [select]:
github.com/prometheus/prometheus/storage/remote.(*QueueManager).updateShardsLoop(0xc120132800)
	/app/storage/remote/queue_manager.go:528 +0x15d
created by github.com/prometheus/prometheus/storage/remote.(*QueueManager).Start
	/app/storage/remote/queue_manager.go:391 +0xb67

goroutine 3439157 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc13a3cdd58)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc13a3cdd50)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc051c72360, 0x2e1c480, 0xc009bf08c0, 0xc2d1d5e300)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 3439416 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc001728b00, 0xc029a3bb00, 0xc03e31b0e0, 0xc02d8678c0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439288 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c000, 0xc045b31080)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920250, 0xc02c80d0a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439421 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc045c4d080, 0xc029a3b6e0, 0xc03e31b620, 0xc03deadb60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3444658 [select]:
net/http.(*persistConn).writeLoop(0xc0c474a480)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4079595 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc049f38790, 0xc045b318c0, 0xc05ca45e60, 0xc02c122c60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3444000 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2ed20, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4085149 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc01cde1900, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc01cde18f0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc01cde18e8, 0xc014fa0e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc01cde18c0, 0xc014fa0e00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc01cde1b80, 0xc0035282a0, 0xc0023ac001)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc01cde1b80, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc01cde1b80, 0x223e960, 0xc0035282a0, 0x203016, 0x203016)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001edb830, 0xc005bd5800, 0x400, 0x400, 0x40, 0x40, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc019775bd0, 0x0, 0x2de7300, 0xc04abeb4c0, 0x21cf51cc4bb3bfb0, 0x5e95aa15, 0xc001756e58, 0x486d66, 0x5e95aa15)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc003528280, 0x8, 0xc001756e50, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc04abeb480)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3443998 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2ebe0, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3445468 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71688, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001d88f98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001d88f80, 0xc04917c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001d88f80, 0xc04917c000, 0x1000, 0x1000, 0xc053cec780, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004d2f480, 0xc04917c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc04a7a7560, 0xc04917c000, 0x1000, 0x1000, 0xc213eb2240, 0xc022b8ac20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc048ce7f20)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc048ce7f20, 0x1, 0x0, 0x0, 0x1, 0xc03ac13300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04a7a7560)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439395 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x7f4ed7f711f0, 0x0, 0xc12e6f2e00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc001899dd8, 0x7f4ecffd9ce0, 0x8)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0015b9740)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001899f40, 0xdf8475800, 0x0, 0x42eb01, 0xc045b317a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc05830f180)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a7480, 0xc017d251b0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439162 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0940, 0xc03be939a0, 0xc2d1d5e3c0)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 4081002 [select]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0501778c0, 0xc3548d0a20, 0xc2ddd0d320, 0xc0154133e0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3445469 [select]:
net/http.(*persistConn).writeLoop(0xc04a7a7560)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439366 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30a80, 0xc0e1598080)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 4083323 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0017288f0, 0xc3548d0a80, 0xc34a6fc060, 0xc04cf8e3c0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439417 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc050177340, 0xc029a3b740, 0xc03e31b260, 0xc03deac480)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3443997 [select, 2 minutes]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2eb40, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 3439109 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a80, 0xc002947680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 4074208 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed53a7e58, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0015a9198, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0015a9180, 0xc05b210000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0015a9180, 0xc05b210000, 0x1000, 0x1000, 0xc04cef9a01, 0x10000c04cef9a80, 0x40c61c0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0016dbf00, 0xc05b210000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc05536ea90, 0xc05b210000, 0x1000, 0x1000, 0x72, 0x0, 0xc056bab958)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc05536ea80, 0xc05b210000, 0x1000, 0x1000, 0x10a2a6780141d, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc056bab950, 0xc05b210000, 0x1000, 0x1000, 0xc04cef9c38, 0x1e8711a, 0xc056bab8f0)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc056b78780)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc056b78780, 0x4, 0x10a2a6780141d, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc059f335e0, 0x2e1c480, 0xc00b360980)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 4090967 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed5b62dc8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0020ebe18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0020ebe00, 0xc04589c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0020ebe00, 0xc04589c000, 0x1000, 0x1000, 0xc011f0f900, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00b7f4068, 0xc04589c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc002c5ed80, 0xc04589c000, 0x1000, 0x1000, 0xc052de43c0, 0xc001767c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc040373b00)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc040373b00, 0x1, 0x0, 0x0, 0x1, 0xc052de4300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002c5ed80)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 4090981 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc04143eca0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc04143ec90)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc04143ec88, 0xc050ee7800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc04143ec60, 0xc050ee7800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc04143ef20, 0xc0417d0400, 0x1)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc04143ef20, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc04143ef20, 0x223e960, 0xc0417d0400, 0xc058763430, 0x70617b)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0417ce780, 0xc10fb45400, 0x400, 0x400, 0x0, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc04119f180, 0x0, 0x2de7300, 0xc001516540, 0x219df53146faef60, 0x5e95ab91, 0xc058763658, 0x486d66, 0x5e95ab91)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0417d03e0, 0x6, 0xc058763700, 0x0, 0xc048d266c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc001516500)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439177 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0ddeb4580)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c370, 0xc002606410)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3421376 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0517e7dc0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0517e7dc0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439430 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc001728c60, 0xc029a3bb00, 0xc2dde42a20, 0xc032f79140)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439161 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc138385808)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc138385800)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0036cd0e0, 0x2e1c480, 0xc009bf0940, 0xc2d1d5e3c0)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 4090591 [select]:
net/http.(*persistConn).writeLoop(0xc008468900)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3438972 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049f380d8, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049f380c8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc049f380b0, 0xc001a9d540, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc002515480)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc002134e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc002134e00, 0x3b9aca00, 0x0, 0xc26d321b01, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc002515480, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00381ed80, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 6729 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc019fc0620, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc019fc0620)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439079 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc049f38658, 0x6c)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc049f38648)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc049f38630, 0xc0021f6390, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0e1598580)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc002820e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc002820e00, 0x3b9aca00, 0x0, 0xc002baa201, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0e1598580, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003823cb0, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 3439372 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c4f0070, 0xc01588a480)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2840944a0, 0xc360366360)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439335 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0d8358400)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4983e0, 0xc02cc00dc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439347 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30a80, 0xc0ddeb5280)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3177828 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc002b9e460, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc002b9e460)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4090812 [select]:
net/http.(*persistConn).writeLoop(0xc00430cea0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3177830 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb80000, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb80000)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439271 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b740, 0xc0d8358900)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 1357524 [select]:
net/http.(*persistConn).writeLoop(0xc00fd68fc0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4089534 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0643690c0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0643690b0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0643690a8, 0xc010b5d600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc064369080, 0xc010b5d600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc064369340, 0xc05ca39540, 0x20302600000001)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc064369340, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc064369340, 0x223e960, 0xc05ca39540, 0x203001, 0x203001)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc05ca358f0, 0xc014437400, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc05b72dae0, 0x0, 0x2de7300, 0xc008d7ecc0, 0x21f11c5c01d66780, 0x5e95ab2d, 0xc023d52658, 0x486d66, 0x5e95ab2d)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc05ca39520, 0x6, 0xc023d52700, 0x441c11, 0xc10a48f800, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc008d7ec80)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1357672 [select]:
net/http.(*persistConn).writeLoop(0xc0026559e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439173 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30d80, 0xc001f47a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439303 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001728e70, 0x2de7680, 0xc00af68c80, 0xc00f79fbf8, 0xc0d2944300, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001728e70, 0xc0522e6a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc200dd9ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00f79fef0, 0x3b9aca00, 0x0, 0xc0021fe701, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001728e70, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e08a0, 0xc0b1e54d00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4088350 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc1048a6f60, 0x5d)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc1048a6f50)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc1048a6f48, 0xc0d15bb801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc1048a6f20, 0xc0d15bb801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc1048a71e0, 0xc04426dd0a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc1048a71e0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc1048a71e0, 0x223e960, 0xc04426dda0, 0x0, 0x0)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc104f43470, 0xc0a268a400, 0x400, 0x400, 0x2de4dc0, 0x0, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc103d23ea0, 0x0, 0x2de7300, 0xc00f6b4cc0, 0x0, 0x0, 0x0, 0xc2f6646238, 0x4590b0)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc104ff93e0, 0xc1fe90ffa8, 0x8, 0x2de4dc0, 0xc0121863c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc13ea3b980)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 164980 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc0500d4700, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc0500d4700)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3440184 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc002c68510, 0xc0000000b4)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc002c68500)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc0343c5440, 0x0, 0x0, 0xc09ab29d00)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0014b5e80, 0x2e1c480, 0xc009bf0980, 0xc2d1d5e420, 0xc041b51400)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0014b5e80, 0x2e1c480, 0xc009bf0980, 0xc2d1d5e420)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3439356 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc3548d0a20, 0xc0ddeb5680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 1357406 [select]:
net/http.(*persistConn).writeLoop(0xc0054c1e60)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357549 [select]:
net/http.(*persistConn).writeLoop(0xc0010099e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439180 [chan receive, 2 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc0c3e43f38, 0x3, 0x8)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc01d533dd8, 0x4313fc, 0xc0c3e43dd0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0c3e43f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01d533f40, 0xdf8475800, 0x0, 0xc0c3e43f01, 0xc021124d20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0507bd900)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c4f00d0, 0xc002606440)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439185 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc283920038)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc283920030)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b47e0, 0x2e1c480, 0xc009bf0a80, 0xc2d1d5e5a0)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 3439182 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc051c48c00, 0x800, 0x6)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc00310add8, 0xc002200e18, 0x4e3f56)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc002200f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00310af40, 0xdf8475800, 0x0, 0xc002606101, 0xc021124d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0ddeb4600)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c30c300, 0xc002606460)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439346 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c30c1c0, 0xc045b31320)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920550, 0xc02c80d960)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439174 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00172a6e0, 0x2de7680, 0xc1fbfdc940, 0xc13e547bf8, 0xc0243b4b40, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00172a6e0, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc201117ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc13e547ef0, 0x3b9aca00, 0x0, 0xc002200f01, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00172a6e0, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0d5dc38d0, 0xc03b534e60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439229 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0d8358380, 0x2e1c480, 0xc009bf0a40, 0xc2d1d5e540)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc12a3d2490, 0x2e1c480, 0xc009bf0a40, 0xc2d1d5e540, 0x2dcba20, 0xc0d8358380)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3439399 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0xc0158bce08, 0x40c1d6, 0xc05a8dba60)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc0017c5dd8, 0xc00f8458c0, 0x438a7e)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0158bcf40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0017c5f40, 0xdf8475800, 0x0, 0x2a5fc01, 0xc045b31860)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0d8358180)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498290, 0xc017d251f0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3444982 [select, 2 minutes]:
net/http.(*persistConn).writeLoop(0xc029042b40)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357483 [select]:
net/http.(*persistConn).writeLoop(0xc012c378c0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439406 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc049f38420, 0xc045b30a80, 0xc045b31bc0, 0xc03be7cc60)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439297 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3bb00, 0xc002947e80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439700 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc050177760, 0xc029a3b740, 0xc136462420, 0xc03fff0660)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439117 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001728b00, 0x2de7680, 0xc000227d40, 0xc144817bf8, 0xc02d8678c0, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001728b00, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc005183ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc144817ef0, 0x3b9aca00, 0x0, 0xc00ed1bf01, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001728b00, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e05a0, 0xc04a455d60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357727 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4ed50, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0ddeb4198, 0x72, 0x3700, 0x37bc, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0ddeb4180, 0xc00f3eb000, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0ddeb4180, 0xc00f3eb000, 0x37bc, 0x37bc, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00663d6c8, 0xc00f3eb000, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc0ca1ffe40, 0xc00f3eb000, 0x37bc, 0x37bc, 0x4, 0x8, 0xc0017168a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc09a153e58, 0x2dca560, 0xc0ca1ffe40, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc09a153c00, 0x7f4ed5b388a0, 0xc008b84f00, 0x5, 0xc008b84f00, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc09a153c00, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc09a153c00, 0xc0a4098000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc04f08ec60, 0xc0a4098000, 0x1000, 0x1000, 0xc054f22f00, 0xc001716c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a4090c00)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a4090c00, 0x1, 0x0, 0x0, 0x1, 0xc00b2afb00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc04f08ec60)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439414 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001729340, 0x2de7680, 0xc1321f1400, 0xc13e54dbf8, 0xc03be7d800, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001729340, 0xc045b318c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00300aef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc13e54def0, 0x3b9aca00, 0x0, 0xc023d54701, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001729340, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0d30, 0xc35560c860)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439314 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b6e0, 0xc05830fe00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439290 [select, 6 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0179182c0, 0x2de7680, 0xc12d8c0140, 0xc26be0bbf8, 0xc03c45b800, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0179182c0, 0xc045b30780, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001920ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc26be0bef0, 0x3b9aca00, 0x0, 0xc002203701, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0179182c0, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc283920290, 0xc02c80d1c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439317 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc049f38160, 0xc3548d0a80, 0xc213eb2180, 0xc03be22e40)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439540 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc050177b80, 0xc045b30d80, 0xc01588b0e0, 0xc03c45a960)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439289 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b30780, 0xc0ddeb4a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439243 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc03be7c480)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 3439412 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03c4f00e0, 0xc03e31af00)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0cf0, 0xc35560c7a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4091013 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc044ea90c0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc044ea90b0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc044ea90a8, 0xc0234c8200, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc044ea9080, 0xc0234c8200, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc044ea9340, 0xc045b9dfe0, 0x2a5faa8)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc044ea9340, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc044ea9340, 0x223e960, 0xc045b9dfe0, 0xc0caa49d98, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc045b97e60, 0xc097551400, 0x400, 0x400, 0x0, 0xc0020efa00, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0440cfb80, 0x0, 0x2de7300, 0xc00af68cc0, 0x40de4e0, 0xc0caa49e70, 0x1e8826a, 0xc004d2f010, 0xc04979e5e1)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc045b9dfc0, 0x0, 0x2dccd20, 0x40906b, 0xc00162e000, 0x4313fc, 0xc0caa49ed0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00af68c80)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3177938 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00dc6c7e0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00dc6c7e0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439413 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc045b318c0, 0xc126a7ef80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439266 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3bb00, 0xc0d8358500)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439595 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc050177a20, 0xc3548d0a20, 0xc03b150de0, 0xc03e05c960)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3424377 [sleep]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
time.Sleep(0x3b9aca00)
	/usr/local/go/src/runtime/time.go:105 +0x157
github.com/prometheus/prometheus/storage/remote.(*shards).sendSamplesWithBackoff(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0xc0360ee000, 0xa67, 0x1388, 0xc2fa83fea8, 0x7b1140, 0xc00049c380)
	/app/storage/remote/queue_manager.go:886 +0x13b
github.com/prometheus/prometheus/storage/remote.(*shards).sendSamples(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0xc0360ee000, 0xa67, 0x1388, 0xc022b86ea8)
	/app/storage/remote/queue_manager.go:838 +0xb3
github.com/prometheus/prometheus/storage/remote.(*shards).runShard(0xc12032b8c0, 0x2e1c480, 0xc11e164ec0, 0x1, 0xc04ff644e0)
	/app/storage/remote/queue_manager.go:827 +0xa98
created by github.com/prometheus/prometheus/storage/remote.(*shards).start
	/app/storage/remote/queue_manager.go:704 +0x272

goroutine 4072143 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed5b62c28, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc104f4bc98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc104f4bc80, 0xc005f9f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc104f4bc80, 0xc005f9f000, 0x1000, 0x1000, 0xc04d099a01, 0x10000c04d099a80, 0x40c6140)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004d2f908, 0xc005f9f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc005aef350, 0xc005f9f000, 0x1000, 0x1000, 0x72, 0x0, 0xc1207dc878)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc005aef340, 0xc005f9f000, 0x1000, 0x1000, 0x10a2bf9f53043, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc1207dc870, 0xc005f9f000, 0x1000, 0x1000, 0xc04d099c38, 0x1e8711a, 0xc1207dc810)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc02703fc80)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc02703fc80, 0x4, 0x10a2bf9f53043, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc005d241e0, 0x2e1c480, 0xc1206d7c00)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 3440193 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00be05210, 0xc1000000ba)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00be05200)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc031eebda0, 0x0, 0x0, 0xc09fc78700)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc002514e80, 0x2e1c480, 0xc009bf08c0, 0xc2d1d5e300, 0xc11cb76600)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc002514e80, 0x2e1c480, 0xc009bf08c0, 0xc2d1d5e300)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 3440244 [select]:
net/http.(*persistConn).writeLoop(0xc07a200d80)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3438894 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0179180d8, 0x5ae0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0179180c8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0179180b0, 0xc017d24330, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb4680)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc015437e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc015437e00, 0x3b9aca00, 0x0, 0xc003928301, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb4680, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e090, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 3177831 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc002b9efc0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc002b9efc0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 4090879 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d4f300, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc001743c18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001743c00, 0xc01595b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc001743c00, 0xc01595b000, 0x1000, 0x1000, 0xc00eb56140, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc010c153b0, 0xc01595b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0025f4a20, 0xc01595b000, 0x1000, 0x1000, 0xc2c0f57200, 0xc0490e9c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc03290b320)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc03290b320, 0x1, 0x0, 0x0, 0x1, 0xc032367200, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0025f4a20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439163 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc12a3d2418)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc12a3d2410)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b4120, 0x2e1c480, 0xc009bf0980, 0xc2d1d5e420)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 4084493 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc001728e70, 0xc0522e6a80, 0xc04d3e8240, 0xc0d2944300)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3443999 [select, 2 minutes]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2ec80, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 1357740 [select]:
net/http.(*persistConn).writeLoop(0xc0b4118120)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439355 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a7570, 0xc045b31560)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc2839206d0, 0xc02c80ddc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439167 [semacquire, 778 minutes]:
sync.runtime_Semacquire(0xc12a3d2498)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc12a3d2490)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc0534b45a0, 0x2e1c480, 0xc009bf0a40, 0xc2d1d5e540)
	/app/discovery/kubernetes/kubernetes.go:465 +0x5ae
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:230 +0x35a

goroutine 4086268 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0d0cf9a60, 0x270)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0d0cf9a50)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0d0cf9a48, 0xc06061b801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0d0cf9a20, 0xc06061b801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0d0cf9ce0, 0xc04699710a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0d0cf9ce0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0d0cf9ce0, 0x223e960, 0xc046997160, 0x0, 0x0)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0de49b740, 0xc0af286000, 0x400, 0x400, 0x2de4dc0, 0x0, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0de49e5f0, 0x0, 0x2de7300, 0xc00f849540, 0x0, 0x0, 0x0, 0xc0367ddaf8, 0x4590b0)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0de499740, 0xc0016edfa8, 0x8, 0x2de4dc0, 0xc008bdde00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc125af7140)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439067 [sync.Cond.Wait, 24 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc045c4cff8, 0x9c)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc045c4cfe8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc045c4cfd0, 0xc001db64d0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc05830fe00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc05d1b7e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc05d1b7e00, 0x3b9aca00, 0x0, 0xc038c11001, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc05830fe00, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc001814090, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 3439298 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc001728c60, 0x2de7680, 0xc1fbfdc780, 0xc140c69bf8, 0xc032f79140, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001728c60, 0xc029a3bb00, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc015434ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc140c69ef0, 0x3b9aca00, 0x0, 0xc02bfc2f01, 0xc029a3bb00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001728c60, 0xc029a3bb00)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0720, 0xc0b1e54380)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 4088353 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f4ed53a7cb8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0f4d0df98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0f4d0df80, 0xc04dfca000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0f4d0df80, 0xc04dfca000, 0x1000, 0x1000, 0xc201e37a01, 0x10000c201e37a80, 0x40c5f40)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc026f23660, 0xc04dfca000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0f1ec4e10, 0xc04dfca000, 0x1000, 0x1000, 0x72, 0x0, 0xc1063b29f8)
	/app/vendor/github.com/soheilhy/cmux/buffer.go:53 +0x12c
github.com/soheilhy/cmux.(*MuxConn).Read(0xc0f1ec4e00, 0xc04dfca000, 0x1000, 0x1000, 0x10a2a49708ff1, 0x40c0d60, 0x0)
	/app/vendor/github.com/soheilhy/cmux/cmux.go:260 +0x4f
net/http.(*connReader).Read(0xc1063b29f0, 0xc04dfca000, 0x1000, 0x1000, 0xc201e37c38, 0x1e8711a, 0xc1063b2990)
	/usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc047cfaea0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc047cfaea0, 0x4, 0x10a2a49708ff1, 0x40c0d60, 0x0, 0x0, 0x40c0d60)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc10419b540, 0x2e1c480, 0xc12c996240)
	/usr/local/go/src/net/http/server.go:1915 +0x9e0
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 1357766 [select]:
net/http.(*persistConn).writeLoop(0xc0b410f9e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4087379 [sync.Cond.Wait, 4 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc032475640, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc032475630)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc032475628, 0xc0a2df0600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc032475600, 0xc0a2df0600, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0324758c0, 0xc0f7286620, 0xc001b94401)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0324758c0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0324758c0, 0x223e960, 0xc0f7286620, 0xc03f7301c0, 0x4)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc094257350, 0xc00f9bdc00, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc03258bdb0, 0x0, 0x2de7300, 0xc002900ec0, 0x21fa5edb01d1fc50, 0x5e95aa9c, 0xc001b94658, 0x486d66, 0x5e95aa9c)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0f7286600, 0x8, 0xc001b94650, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002900e00)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 4069349 [select]:
net/http.(*persistConn).writeLoop(0xc0144278c0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439525 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc013ed0630, 0xc045b318c0, 0xc213eb2b40, 0xc03d0102a0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439113 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc002514f80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4b2290, 0xc001a974a0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439337 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0d8358480)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b498370, 0xc02cc00de0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439072 [sync.Cond.Wait]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc050177838, 0x6b)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc050177828)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc050177810, 0xc02cc00e00, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8359500)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0086bfe00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0086bfe00, 0x3b9aca00, 0x0, 0xc0047d0601, 0xc3548d0a20)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8359500, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0018141b0, 0xc3548d0a20)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 186580 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed4d71f78, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc051b24c98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc051b24c80, 0xc0547f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc051b24c80, 0xc0547f6000, 0x1000, 0x1000, 0xc00eb56140, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01b21aa10, 0xc0547f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc054104900, 0xc0547f6000, 0x1000, 0x1000, 0xc0143bc960, 0xc00191fc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0545f2240)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0545f2240, 0x1, 0x0, 0x0, 0x1, 0xc2ed2c2e00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc054104900)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 4089599 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc07f3e2b40, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc07f3e2b30)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc07f3e2b28, 0xc01c382c00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc07f3e2b00, 0xc01c382c00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc07f3e2dc0, 0xc07f464680, 0xc00181e401)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc07f3e2dc0, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc07f3e2dc0, 0x223e960, 0xc07f464680, 0x203004, 0x203004)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc07e7f9080, 0xc075707800, 0x400, 0x400, 0x10, 0x10, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc07f114230, 0x0, 0x2de7300, 0xc010730640, 0x219da18801d91b30, 0x5e95ab37, 0xc00181e658, 0x486d66, 0x5e95ab37)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc07f464660, 0x6, 0xc00181e700, 0x441c11, 0xc1009fe3c0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc010730600)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439066 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc045c4ce98, 0x52)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc045c4ce88)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc045c4ce70, 0xc001db60e0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc05830f980)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001898e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001898e00, 0x3b9aca00, 0x0, 0xc26d8f4901, 0xc029a3b6e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc05830f980, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc001fc3cb0, 0xc029a3b6e0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3439313 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4a7500, 0xc3548d1320)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc138385b20, 0xc0541bc300)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439225 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc0343c55c0)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:58 +0x132

goroutine 1357765 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed53a7a48, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0e1598518, 0x72, 0x3700, 0x37bc, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0e1598500, 0xc00f3ee800, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0e1598500, 0xc00f3ee800, 0x37bc, 0x37bc, 0xd, 0x8, 0x1000)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc01025f750, 0xc00f3ee800, 0x37bc, 0x37bc, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc02c7609c0, 0xc00f3ee800, 0x37bc, 0x37bc, 0x4, 0x8, 0xc05d1b98a0)
	/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0a33373d8, 0x2dca560, 0xc02c7609c0, 0x40a1b5, 0x225d900, 0x23e4020)
	/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0a3337180, 0x7f4ed5b388a0, 0xc00fea2f40, 0x5, 0xc00fea2f40, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0a3337180, 0x0, 0x0, 0x3)
	/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0a3337180, 0xc0a4320000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc0b410f9e0, 0xc0a4320000, 0x1000, 0x1000, 0xc13a03d380, 0xc05d1b9c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a431a8a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a431a8a0, 0x1, 0x0, 0x0, 0x1, 0xc05a462f00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0b410f9e0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439276 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3b740, 0xc0d8358d00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439115 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b498070, 0xc03e31a7e0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0560, 0xc04a455c80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439343 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x1f4)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc01e82ddd8, 0x0, 0xc00170ddf8)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00170df40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01e82df40, 0xdf8475800, 0x0, 0x1, 0xc0522e76e0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc05830f100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a74f0, 0xc02cc010c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439219 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc001728d38, 0x68)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc001728d28)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001728d10, 0xc001a97b20, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc126a7e280)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00191ee00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00191ee00, 0x3b9aca00, 0x0, 0xc01078d501, 0xc0522e6a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc126a7e280, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003122900, 0xc0522e6a80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:361 +0x1e0e

goroutine 1357454 [select]:
net/http.(*persistConn).writeLoop(0xc0026546c0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 1357533 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b97b0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc002578b18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002578b00, 0xc0a38aa000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc002578b00, 0xc0a38aa000, 0x1000, 0x1000, 0xc0029637c0, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc009982b18, 0xc0a38aa000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc002654fc0, 0xc0a38aa000, 0x1000, 0x1000, 0xc035085620, 0xc005168c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0a3888780)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0a3888780, 0x1, 0x0, 0x0, 0x1, 0xc02fbbbb00, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002654fc0)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3438783 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0014b5e80, 0x2e1c480, 0xc009bf0980, 0xc2d1d5e420)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc12a3d2410, 0x2e1c480, 0xc009bf0980, 0xc2d1d5e420, 0x2dcba20, 0xc0014b5e80)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570

goroutine 3177829 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc00fb801c0, 0x2e1c4c0, 0xc000048020)
	/app/rules/manager.go:382 +0x5b1
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc0003f30e0, 0xc00fb801c0)
	/app/rules/manager.go:936 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/app/rules/manager.go:931 +0x56

goroutine 3439179 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0507bd980)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c4f0060, 0xc002606430)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439307 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0522e6a80, 0xc126a7e780)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3438896 [sync.Cond.Wait, 778 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc017918398, 0x1)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc017918388)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc017918370, 0xc017d248d0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0ddeb4e80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc001c25e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001c25e00, 0x3b9aca00, 0x0, 0xc03c30e001, 0xc045b30780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0ddeb4e80, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e120, 0xc045b30780)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 3439209 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc049f382c0, 0x2de7680, 0xc046fac100, 0xc003f11bf8, 0xc03dd2fda0, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc049f382c0, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00281fef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003f11ef0, 0x3b9aca00, 0x0, 0xc001735701, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc049f382c0, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc284094040, 0xc02bd83120)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439081 [sync.Cond.Wait, 148 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0017292b8, 0x17)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0017292a8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001729290, 0xc002168540, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc126a7ef80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc003009e00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc003009e00, 0x3b9aca00, 0x0, 0xc03c524001, 0xc045b318c0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc126a7ef80, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc003823d40, 0xc045b318c0)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:363 +0x1eae

goroutine 1357658 [select]:
net/http.(*persistConn).writeLoop(0xc04f0890e0)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 4083309 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc00172a6e0, 0xc045b30d80, 0xc023ed4ae0, 0xc0243b4b40)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439110 [select, 2 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0017288f0, 0x2de7680, 0xc008dd4bc0, 0xc26be0dbf8, 0xc04cf8e3c0, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0017288f0, 0xc3548d0a80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc025fa3ef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc26be0def0, 0x3b9aca00, 0x0, 0xc023d4e701, 0xc3548d0a80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0017288f0, 0xc3548d0a80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3e0430, 0xc04a455a40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 1357537 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9bc0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc05bbc3098, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc05bbc3080, 0xc095db4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc05bbc3080, 0xc095db4000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc012fb3648, 0xc095db4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc0047f8a20, 0xc095db4000, 0x1000, 0x1000, 0xc02c07a900, 0xc00171bc20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc0944a2b40)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0944a2b40, 0x1, 0x0, 0x0, 0x1, 0xc0015e2300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc0047f8a20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3439202 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc03b4b21c0, 0xc053f24420)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc13a3cdd60, 0xc02bd82600)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3444486 [select]:
net/http.(*persistConn).writeLoop(0xc0b5962240)
	/usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 3439434 [select, 8 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc050177600, 0xc029a3b740, 0xc2dde435c0, 0xc032f79800)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:260 +0x172
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:254 +0x291

goroutine 3439116 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc029a3bb00, 0xc002947a80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439190 [select, 8 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).updater(0xc000441720, 0x2e1c480, 0xc009bf0b40, 0xc03c2435e0, 0xc2d1d5e660)
	/app/discovery/manager.go:236 +0x1b3
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/app/discovery/manager.go:231 +0x3b0

goroutine 3439312 [chan receive, 778 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc0522e6a80, 0xc126a7eb80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:104 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0xae

goroutine 3439344 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc05830f100)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621 +0x18d
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4a74f0, 0xc02cc010d0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439112 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x7f4ed7f718b8, 0x0, 0x10)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc001764dd8, 0xc02b3193b0, 0xc02e3e0e28)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc02e3e0f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001764f40, 0xdf8475800, 0x0, 0x42eb01, 0xc03e31a780)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc002514f80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03b4b2290, 0xc001a97490)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439886 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc010673850, 0xc1000000b7)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc010673840)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/util/workqueue.(*Type).Get(0xc032f78ea0, 0x0, 0x0, 0xc09fc01d00)
	/app/vendor/k8s.io/client-go/util/workqueue/queue.go:145 +0xa0
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process(0xc0507bd880, 0x2e1c480, 0xc0100d0400, 0xc2d1d5f680, 0x0)
	/app/discovery/kubernetes/endpoints.go:153 +0x4f
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc0507bd880, 0x2e1c480, 0xc0100d0400, 0xc2d1d5f680)
	/app/discovery/kubernetes/endpoints.go:144 +0x49
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run
	/app/discovery/kubernetes/endpoints.go:143 +0x373

goroutine 4086653 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc12d8c21a0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc12d8c2190)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc12d8c2188, 0xc055642400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc12d8c2160, 0xc055642400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc12d8c2420, 0xc12d54aec0, 0x2a5faa8)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc12d8c2420, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc12d8c2420, 0x223e960, 0xc12d54aec0, 0xc023d4ed98, 0x40c1d6)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc12d5f6630, 0xc127f35400, 0x400, 0x400, 0x0, 0xc0bb1ed080, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc12d5f82d0, 0x0, 0x2de7300, 0xc12d8c0180, 0x4313fc, 0xc023d4ee30, 0x45be00, 0xc00053fc80, 0x4)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc12d54aea0, 0xc023d4eee8, 0x3, 0x0, 0xc015cd2840, 0xc0bf0f3c80, 0xc0bf0f3d08)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc12d8c0140)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 1357598 [IO wait]:
internal/poll.runtime_pollWait(0x7f4ed59b9af0, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc048343618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc048343600, 0xc0a11e0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc048343600, 0xc0a11e0000, 0x1000, 0x1000, 0xc002963b80, 0x0, 0x4400cc)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00663ca70, 0xc0a11e0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc000fb9c20, 0xc0a11e0000, 0x1000, 0x1000, 0xc051c79bc0, 0xc001886c20, 0x404d15)
	/usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc09a2075c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc09a2075c0, 0x1, 0x0, 0x0, 0x1, 0xc002c88300, 0x0)
	/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc000fb9c20)
	/usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 3444449 [select]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc0b3e2edc0, 0xdf8475800, 0x2540be400, 0x0)
	/app/scrape/scrape.go:991 +0x87e
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/app/scrape/scrape.go:422 +0x542

goroutine 4084367 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0c670e9e0, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0c670e9d0)
	/usr/local/go/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0c670e9c8, 0xc05b20da00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc0c670e9a0, 0xc05b20da00, 0x200, 0x200, 0x0, 0x0, 0x0)
	/app/vendor/golang.org/x/net/http2/transport.go:2019 +0xac
encoding/json.(*Decoder).refill(0xc0c670ec60, 0xc0bcbdfe40, 0x2a5fa01)
	/usr/local/go/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc0c670ec60, 0x0, 0x0, 0x21fbde0)
	/usr/local/go/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc0c670ec60, 0x223e960, 0xc0bcbdfe40, 0x203018, 0x203018)
	/usr/local/go/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0c5d03350, 0xc03f919c00, 0x400, 0x400, 0x8, 0x8, 0x38)
	/app/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0c7f0e0f0, 0x0, 0x2de7300, 0xc046fac140, 0x219f4f774d0aa300, 0x5e95a9de, 0xc0c3e44658, 0x486d66, 0x5e95a9de)
	/app/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0bcbdfe20, 0x8, 0xc0c3e44650, 0x1d78c2a, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc046fac100)
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/app/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 3439245 [sync.Cond.Wait, 8 minutes]:
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc050177af8, 0x4e)
	/usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc050177ae8)
	/usr/local/go/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc050177ad0, 0xc02cc013f0, 0x0, 0x0, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0d8359d00)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:153 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00310de00)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00310de00, 0x3b9aca00, 0x0, 0xc26dd72901, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc0d8359d00, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/controller.go:125 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc002f4e360, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331 +0x47f
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:362 +0x1e5e

goroutine 3439178 [chan receive, 8 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x1, 0x0, 0x1f4)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653 +0x54
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc01a214dd8, 0x0, 0xc000631df8)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292 +0x51
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652 +0x79
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000631f40)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc01a214f40, 0xdf8475800, 0x0, 0x1, 0xc021124cc0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0507bd980)
	/app/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650 +0x9b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc03c4f0060, 0xc002606420)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3439387 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc050177b80, 0x2de7680, 0xc001516500, 0xc200ff5bf8, 0xc03c45a960, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:354 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc050177b80, 0xc045b30d80, 0x0, 0x0)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:320 +0x8b1
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:152 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00310aef0)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc200ff5ef0, 0x3b9aca00, 0x0, 0xc0158bef01, 0xc045b30d80)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc050177b80, 0xc045b30d80)
	/app/vendor/k8s.io/client-go/tools/cache/reflector.go:151 +0x198
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc12a3d2f00, 0xc35fd5ca60)
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 3424380 [select, 790 minutes]:
github.com/prometheus/prometheus/storage/remote.(*QueueManager).reshardLoop(0xc120132800)
	/app/storage/remote/queue_manager.go:640 +0x142
created by github.com/prometheus/prometheus/storage/remote.(*QueueManager).Start
	/app/storage/remote/queue_manager.go:392 +0xb89

goroutine 3439247 [chan receive, 778 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc0ddeb4500, 0x2e1c480, 0xc009bf0b40, 0xc2d1d5e660)
	/app/discovery/kubernetes/endpoints.go:149 +0x3a4
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15(0xc2839200b0, 0x2e1c480, 0xc009bf0b40, 0xc2d1d5e660, 0x2dcba20, 0xc0ddeb4500)
	/app/discovery/kubernetes/kubernetes.go:459 +0x8c
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/app/discovery/kubernetes/kubernetes.go:457 +0x570