gistfile1.txt

goroutine 1234 [running]:
runtime/pprof.writeGoroutineStacks(0x7f9a091736b0, 0xc4292dbc80, 0x4106a9, 0xc4243bb3e0)
	/usr/local/go/src/runtime/pprof/pprof.go:650 +0xa7
runtime/pprof.writeGoroutine(0x7f9a091736b0, 0xc4292dbc80, 0x2, 0xc4216e4000, 0x1c39dc0)
	/usr/local/go/src/runtime/pprof/pprof.go:639 +0x44
runtime/pprof.(*Profile).WriteTo(0x286c900, 0x7f9a091736b0, 0xc4292dbc80, 0x2, 0xc4292dbc80, 0x16)
	/usr/local/go/src/runtime/pprof/pprof.go:310 +0x3e4
net/http/pprof.handler.ServeHTTP(0xc4291cd40d, 0x9, 0x1c47420, 0xc4292dbc80, 0xc4246c6a00)
	/usr/local/go/src/net/http/pprof/pprof.go:237 +0x1b8
net/http/pprof.Index(0x1c47420, 0xc4292dbc80, 0xc4246c6a00)
	/usr/local/go/src/net/http/pprof/pprof.go:248 +0x1d1
github.com/prometheus/prometheus/web.serveDebug(0x1c47420, 0xc4292dbc80, 0xc4246c6a00)
	/go/src/github.com/prometheus/prometheus/web/web.go:316 +0x1be
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc4292dbc80, 0xc4246c6900, 0xc4292dbca0, 0x1, 0x1)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc42016aac0, 0x1c47420, 0xc4292dbc80, 0xc4246c6900)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4201264a0, 0x1c47420, 0xc4292dbc80, 0xc4246c6900)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc4292dbc80, 0xc4246c6900)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc4210536c0, 0xc4246c6800)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc4210536c0, 0xc4246c6800)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc4210536c0, 0xc4246c6800)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc427560aa0, 0x1c50c20, 0xc4248c9500)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run(0xc4204c1c88, 0xc4205aeeb0, 0x8)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:43 +0xec
main.main()
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:581 +0x5237

goroutine 17 [syscall, 2 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 19 [chan receive]:
github.com/prometheus/prometheus/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x2883600)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:879 +0x8b
created by github.com/prometheus/prometheus/vendor/github.com/golang/glog.init.0
	/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:410 +0x203

goroutine 179 [select, 2 minutes, locked to thread]:
runtime.gopark(0x1b6e570, 0x0, 0x1ade6e1, 0x6, 0x18, 0x1)
	/usr/local/go/src/runtime/proc.go:291 +0x11a
runtime.selectgo(0xc420476750, 0xc420184720)
	/usr/local/go/src/runtime/select.go:392 +0xe50
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal_unix.go:549 +0x1f4
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1

goroutine 197 [select]:
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*SegmentWAL).run(0xc4202761e0, 0x2540be400)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/wal.go:704 +0x36e
created by github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.OpenSegmentWAL
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/wal.go:244 +0x776

goroutine 172 [chan receive (nil chan), 2 minutes]:
github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1.1(0x7f9a09468158, 0xc420044018, 0xc4200c6340, 0x7ffcf52686d2, 0x3)
	/go/src/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go:85 +0x4c
created by github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1
	/go/src/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go:84 +0x19b

goroutine 180 [select, 2 minutes]:
main.main.func6(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:372 +0x121
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc4204a3b40, 0xc4202fe010)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 181 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc4205736c0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:93 +0x50
main.main.func8(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:393 +0x40
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc42000a020, 0xc42000a080)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 182 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery.(*Manager).Run(0xc420573730, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:93 +0x50
main.main.func10(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:406 +0x40
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc42000a160, 0xc42000a1a0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 183 [semacquire]:
sync.runtime_Semacquire(0xc421cfe34c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc421cfe340)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/scrape.(*scrapePool).sync(0xc420fc0700, 0xc425128000, 0x32, 0x40)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:331 +0x620
github.com/prometheus/prometheus/scrape.(*scrapePool).Sync(0xc420fc0700, 0xc422d2ce00, 0x38, 0x40)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:270 +0x5d6
github.com/prometheus/prometheus/scrape.(*Manager).reload(0xc42017d360, 0xc4256c6d80)
	/go/src/github.com/prometheus/prometheus/scrape/manager.go:164 +0x187
github.com/prometheus/prometheus/scrape.(*Manager).Run(0xc42017d360, 0xc420184360, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/manager.go:62 +0x4c
main.main.func12(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:425 +0x7a
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc42025e030, 0xc420126200)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 184 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc4206f1cf4)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc4206f1ce8)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/storage/remote.(*shards).stop(0xc4206f1cc0)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:406 +0x74
github.com/prometheus/prometheus/storage/remote.(*QueueManager).Stop(0xc4201f0b40)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:258 +0x15a
github.com/prometheus/prometheus/storage/remote.(*Storage).ApplyConfig(0xc420038f60, 0xc424e50380, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:81 +0x3c5
github.com/prometheus/prometheus/storage/remote.(*Storage).ApplyConfig-fm(0xc424e50380, 0x1e, 0xc424e50380)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:301 +0x34
main.reloadConfig(0x7ffcf526869e, 0x1e, 0x1c370e0, 0xc420393b60, 0xc420046d80, 0x7, 0x7, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:606 +0x236
main.main.func14(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:454 +0x203
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc4203ca960, 0xc4205aee80)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 185 [chan receive, 2 minutes]:
main.main.func16(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:492 +0x325
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc4203ca9c0, 0xc4205aee90)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 186 [chan receive, 2 minutes]:
main.main.func18(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:519 +0x41d
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc420046dc0, 0xc4202c6000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 187 [select, 2 minutes]:
github.com/prometheus/prometheus/web.(*Handler).Run(0xc42033ea00, 0x1c50c20, 0xc42016a400, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/web/web.go:470 +0xe6b
main.main.func20(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:533 +0x40
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc42060e600, 0xc4205aeea0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 188 [chan receive, 2 minutes]:
main.main.func22(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:552 +0x4e
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc42060e620, 0xc42060e640)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 189 [select]:
github.com/prometheus/prometheus/notifier.(*Manager).Run(0xc420530680, 0xc4201843c0)
	/go/src/github.com/prometheus/prometheus/notifier/notifier.go:303 +0x133
main.main.func24(0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:572 +0x7a
github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run.func1(0xc4203caa20, 0xc4202c62a0, 0xc4205aeeb0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:38 +0x27
created by github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group.(*Group).Run
	/go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/oklog/pkg/group/group.go:37 +0xa8

goroutine 171 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4200c64e0)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/clientconn.go:908 +0x1c0
github.com/prometheus/prometheus/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc4200c64e0)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/clientconn.go:637 +0x1af
created by github.com/prometheus/prometheus/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/clientconn.go:628 +0x6d8

goroutine 269 [chan receive]:
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420126640, 0x1b6dee0, 0xc428f5d360, 0x1c50ce0, 0xc420f44090)
	<autogenerated>:1 +0x5b
net/http.(*Server).Serve(0xc42053e820, 0x1c4f320, 0xc420126640, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:2770 +0x1a5
github.com/prometheus/prometheus/web.(*Handler).Run.func5(0xc42053e820, 0x1c4f320, 0xc420126640, 0xc42033ea00)
	/go/src/github.com/prometheus/prometheus/web/web.go:455 +0x43
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/go/src/github.com/prometheus/prometheus/web/web.go:454 +0xced

goroutine 191 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f9a094a4e30, 0x72, 0xc420c18b88)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420410498, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420410498, 0xc420bf0000, 0x8000, 0x8000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420410480, 0xc420bf0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420410480, 0xc420bf0000, 0x8000, 0x8000, 0x60, 0x1a530a0, 0x1)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42039e1d8, 0xc420bf0000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
bufio.(*Reader).Read(0xc4203cade0, 0xc420c02038, 0x9, 0x9, 0xc420470fa0, 0x2, 0x2)
	/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x1c36800, 0xc4203cade0, 0xc420c02038, 0x9, 0x9, 0x9, 0x2, 0xc4200c64e0, 0xc420470dd8)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x1c36800, 0xc4203cade0, 0xc420c02038, 0x9, 0x9, 0x403f2c, 0xc4201fa780, 0x4)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.readFrameHeader(0xc420c02038, 0x9, 0x9, 0x1c36800, 0xc4203cade0, 0x0, 0xc400000000, 0x0, 0x2)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420c02000, 0xc4204a3e40, 0x0, 0xc4201849c0, 0xc420470fa0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport.(*framer).readFrame(0xc4202c6570, 0xc420470fa0, 0x0, 0xc4201849c0, 0x53d70d)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport/http_util.go:608 +0x2f
github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42009be00)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport/http2_client.go:1080 +0x47
created by github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport/http2_client.go:267 +0xb6c

goroutine 192 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport.(*http2Client).controller(0xc42009be00)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport/http2_client.go:1168 +0x122
created by github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/transport/http2_client.go:297 +0xca2

goroutine 270 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:184
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420126480, 0x1b6a318, 0xc420281e60, 0x1c4f320, 0xc420126480)
	<autogenerated>:1 +0x5b
github.com/prometheus/prometheus/vendor/google.golang.org/grpc.(*Server).Serve(0xc420281e60, 0x1c4f320, 0xc420126480, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/server.go:432 +0x16c
github.com/prometheus/prometheus/web.(*Handler).Run.func6(0xc420281e60, 0x1c4f320, 0xc420126480, 0xc42033ea00)
	/go/src/github.com/prometheus/prometheus/web/web.go:460 +0x43
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/go/src/github.com/prometheus/prometheus/web/web.go:459 +0xd39

goroutine 271 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4f00, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4205b4118, 0x72, 0xc4204a3e00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4205b4118, 0xffffffffffffff00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc4205b4100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:372 +0x1a8
net.(*netFD).accept(0xc4205b4100, 0x1c59d40, 0xc4200a4298, 0x1adc017)
	/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc420494008, 0xc4200a4240, 0xc4209c2e98, 0x1)
	/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc420494008, 0xc4209c2e98, 0x1955400, 0x1955400, 0xc4244f3a10)
	/usr/local/go/src/net/tcpsock.go:259 +0x49
github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil.(*limitListener).Accept(0xc4201263a0, 0x4346f4, 0xc4209c2ee8, 0x4571a0, 0xc4209c2f28)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil/listen.go:30 +0x53
github.com/prometheus/prometheus/vendor/github.com/mwitkow/go-conntrack.(*connTrackListener).Accept(0xc420126460, 0x1b67740, 0xc4204a3f40, 0x1c5d880, 0xc4244f3b60)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go:86 +0x37
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4204a3f40, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x88
github.com/prometheus/prometheus/web.(*Handler).Run.func7(0xc4200a4420, 0x1c45d60, 0xc4204a3f40)
	/go/src/github.com/prometheus/prometheus/web/web.go:467 +0x31
created by github.com/prometheus/prometheus/web.(*Handler).Run
	/go/src/github.com/prometheus/prometheus/web/web.go:466 +0xd8e

goroutine 272 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f9a094a4d60, 0x72, 0xc4209c3940)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4205b4398, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4205b4398, 0xc4203d0100, 0x9, 0x9)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4205b4380, 0xc4203d0118, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4205b4380, 0xc4203d0118, 0x9, 0x9, 0x4, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4204948e0, 0xc4203d0118, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*bufferedReader).Read(0xc4200bc7f8, 0xc4203d0118, 0x9, 0x9, 0x100000000000000, 0x0, 0x10)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/buffer.go:42 +0x120
io.ReadAtLeast(0x1c37020, 0xc4200bc7f8, 0xc4203d0118, 0x9, 0x9, 0x9, 0x410ef8, 0x10, 0x19641c0)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x1c37020, 0xc4200bc7f8, 0xc4203d0118, 0x9, 0x9, 0xf7e2bca566869f01, 0xefff100000004, 0x3)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.readFrameHeader(0xc4203d0118, 0x9, 0x9, 0x1c37020, 0xc4200bc7f8, 0x0, 0x0, 0xc4206e65b0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4203d00e0, 0x1c3ea20, 0xc4206e65b0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.matchHTTP2Field(0x1c37020, 0xc4200bc7f8, 0x1ae5879, 0xc, 0x1aea93a, 0x10, 0x7f9a091e46d0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/matchers.go:145 +0x140
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.HTTP2HeaderField.func1(0x1c37020, 0xc4200bc7f8, 0xc4209baea0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/matchers.go:111 +0x59
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*cMux).serve(0xc4204a3f40, 0x1c5d880, 0xc420f1e420, 0xc4200a42a0, 0xc4206e6470)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:143 +0x1f3
created by github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*cMux).Serve
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:133 +0x15d

goroutine 299 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc42055801c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420558010)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc420047140, 0x1c50c20, 0xc420047180, 0xc4200a4900)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:307 +0x47e
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 292 [select]:
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*DB).run(0xc42015c300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/db.go:265 +0x25c
created by github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.Open
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/db.go:243 +0x5ba

goroutine 853 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119f88, 0x72, 0xc426df6960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261d7918, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261d7918, 0xc425e7e000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261d7900, 0xc425e7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261d7900, 0xc425e7e000, 0x1000, 0x1000, 0xc421eb22f8, 0xc426116360, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4204948e8, 0xc425e7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc426116360, 0xc425e7e000, 0x1000, 0x1000, 0xc426df6b98, 0x403855, 0xc4261e9da0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4204131a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4204131a0, 0x1, 0x0, 0x0, 0x0, 0xc4261e9c20, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc426116360)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 849 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f9a092d5680, 0x72, 0xc426cab960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc426fc2098, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc426fc2098, 0xc426cb3000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc426fc2080, 0xc426cb3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc426fc2080, 0xc426cb3000, 0x1000, 0x1000, 0xc426dcd138, 0xc4241fd560, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426de4018, 0xc426cb3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4241fd560, 0xc426cb3000, 0x1000, 0x1000, 0xc426cabb98, 0x403855, 0xc426453380)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc426dea060)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc426dea060, 0x1, 0x0, 0x0, 0x0, 0xc4264532c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4241fd560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 295 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/storage/remote.(*shards).runShard(0xc4206f1cc0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:454 +0x3b5
created by github.com/prometheus/prometheus/storage/remote.(*shards).start
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:398 +0x48

goroutine 296 [select]:
github.com/prometheus/prometheus/storage/remote.(*QueueManager).updateShardsLoop(0xc4201f0d20)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:269 +0x163
created by github.com/prometheus/prometheus/storage/remote.(*QueueManager).Start
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:241 +0x63

goroutine 297 [select, 2 minutes]:
github.com/prometheus/prometheus/storage/remote.(*QueueManager).reshardLoop(0xc4201f0d20)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:345 +0x135
created by github.com/prometheus/prometheus/storage/remote.(*QueueManager).Start
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:242 +0x85

goroutine 298 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/storage/remote.(*shards).runShard(0xc4206f1e40, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:454 +0x3b5
created by github.com/prometheus/prometheus/storage/remote.(*shards).start
	/go/src/github.com/prometheus/prometheus/storage/remote/queue_manager.go:398 +0x48

goroutine 300 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4205d1e00)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc420047180, 0xc420614780, 0xf, 0xc4205d1e50, 0xc, 0xc4200a4900)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 301 [chan send]:
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc420047180)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:161 +0x1b0
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 68 [semacquire]:
sync.runtime_notifyListWait(0xc421248028, 0xe)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421248018)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc421248000, 0xc42031e0d0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc420394000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4209ccf10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4209ccf10, 0x3b9aca00, 0x0, 0x1744b01, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4209ccf10, 0x3b9aca00, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc420394000, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc4200bc820, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296 +0x470

goroutine 44 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc420458480, 0xc420394000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 45 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc420262a20, 0x1c3faa0, 0xc421c00900, 0xc420b9fc80, 0xc421c5c060, 0xc420458480, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc420262a20, 0xc420458480, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc42060e020)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc42060e020, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc42060e020, 0x3b9aca00, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 302 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc42055803c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420558030)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4200478c0, 0x1c50c20, 0xc420047900, 0xc4200a4960)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:307 +0x47e
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 303 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4204fe500)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc420047900, 0xc4205a9a00, 0x1a, 0xc4204fe5a0, 0xc, 0xc4200a4960)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 304 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc420047900)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc420047900)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:159 +0x16a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 70 [semacquire]:
sync.runtime_notifyListWait(0xc42015c268, 0xe)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42015c258)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc42015c240, 0xc420432120, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc420456180)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4209c5f10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4209c5f10, 0x3b9aca00, 0x0, 0x1744b01, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4209c5f10, 0x3b9aca00, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc420456180, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc4200bcd20, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296 +0x470

goroutine 71 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc42128c000, 0xc420456180)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 72 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc4202be1b0, 0x1c3faa0, 0xc420e31650, 0xc4212a1c80, 0xc42018e420, 0xc42128c000, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc4202be1b0, 0xc42128c000, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc421242040)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc421242040, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc421242040, 0x3b9aca00, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 657 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc426dc4cc0, 0x6, 0x6, 0x1620a77b947, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc426dc0b40, 0xc426dc4cc0, 0x6, 0x6, 0x1620a77b947, 0x3ff0000000000000, 0xc426dc4cc0, 0x6, 0x6)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4277f78c0, 0xc426dc4cc0, 0x6, 0x6, 0x1620a77b947, 0x3ff0000000000000, 0x1c36b01, 0xc420084060, 0x4412)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420275d00, 0xc427afe000, 0x1e4412, 0x2a2e00, 0xbea0b8fe49809e88, 0x4b6d22ee2, 0x2883180, 0x6572222c5d7d2274, 0x6c6f507472617473, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420275d00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 367 [semacquire]:
sync.runtime_notifyListWait(0xc421345940, 0xc40000000c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421345930)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc421345928, 0xc4277ee001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421345900, 0xc4277ee001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc421e66000, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc421e66000, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc421e66000, 0x18542c0, 0xc428f23640, 0x1c38800, 0xc427839800)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc421c008d0, 0xc427794600, 0x2000, 0x2600, 0xc428f260c0, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc421e5c050, 0x0, 0x1c37c80, 0xc428f260c0, 0x1a80, 0x1c38800, 0xc421b21fd8, 0x454dc0, 0xc421b21f80)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421b26a00, 0xc42138ffa8, 0x7, 0x1c38800, 0xc427839800, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc421c00900)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 305 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run(0xc4212be0f0, 0x1c50c20, 0xc4212ba0c0, 0xc4200a49c0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:105 +0x46f
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4212ba080, 0x1c50c20, 0xc4212ba0c0, 0xc4200a49c0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:359 +0x8a8
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 306 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4204fec00)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc4212ba0c0, 0xc4205a9ea0, 0x13, 0xc4204fec00, 0xc, 0xc4200a49c0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 307 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4212ba000)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc4212ba0c0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:159 +0x16a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 321 [semacquire]:
sync.runtime_notifyListWait(0xc4212c20e8, 0x3e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4212c20d8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc4212c20c0, 0xc4205fe030, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc42033c180)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc420f4ef10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc420f4ef10, 0x3b9aca00, 0x0, 0x1744b01, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc420f4ef10, 0x3b9aca00, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc42033c180, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc420194780, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:354 +0x83e

goroutine 322 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc42128c2a0, 0xc42033c180)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 323 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc4212e6000, 0x1c3faa0, 0xc4213084e0, 0xc42129dc80, 0xc4205c8060, 0xc42128c2a0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc4212e6000, 0xc42128c2a0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc420182140)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc420182140, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc420182140, 0x3b9aca00, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 854 [select]:
net/http.(*persistConn).writeLoop(0xc426116360)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 308 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run(0xc4212d0270, 0x1c50c20, 0xc4212ba700, 0xc4200a4a20)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:105 +0x46f
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4212ba6c0, 0x1c50c20, 0xc4212ba700, 0xc4200a4a20)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:359 +0x8a8
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 309 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4204ff300)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc4212ba700, 0xc4204ac480, 0x14, 0xc4204ff360, 0xc, 0xc4200a4a20)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 310 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4212ba700)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc4212ba700)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:159 +0x16a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 340 [semacquire]:
sync.runtime_notifyListWait(0xc421320028, 0x3e)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421320018)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc421320000, 0xc420636050, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc42017a080)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc420f4df10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc420f4df10, 0x3b9aca00, 0x0, 0x1744b01, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc420f4df10, 0x3b9aca00, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc42017a080, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc420298320, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:354 +0x83e

goroutine 341 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc420458720, 0xc42017a080)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 342 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc42132a000, 0x1c3faa0, 0xc420393d70, 0xc42133dc80, 0xc4214e2120, 0xc420458720, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc42132a000, 0xc420458720, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4212dc080)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4212dc080, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4212dc080, 0x3b9aca00, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 838 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f9a092d5820, 0x72, 0xc426dfa960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4266a6818, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4266a6818, 0xc426bf5000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4266a6800, 0xc426bf5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4266a6800, 0xc426bf5000, 0x1000, 0x1000, 0xc426dcd148, 0xc4266afd40, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423254b40, 0xc426bf5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4266afd40, 0xc426bf5000, 0x1000, 0x1000, 0xc426dfab98, 0x403855, 0xc4201858c0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421c5d6e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421c5d6e0, 0x1, 0x0, 0x0, 0x0, 0xc420185800, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4266afd40)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 850 [select, 1 minutes]:
net/http.(*persistConn).writeLoop(0xc4241fd560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 311 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc42016e1ac)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc42016e1a0)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4212bad00, 0x1c50c20, 0xc4212bad40, 0xc4200a4a80)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:287 +0xfa2
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 312 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4204ffa00)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc4212bad40, 0xc4204ad2c0, 0x12, 0xc4204ffa00, 0xc, 0xc4200a4a80)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 313 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4212bad00)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc4212bad40)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:159 +0x16a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 327 [semacquire]:
sync.runtime_notifyListWait(0xc4213200e8, 0x89)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4213200d8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc4213200c0, 0xc420636260, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc42017a580)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc421331f10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc421331f10, 0x3b9aca00, 0x0, 0x1744b01, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc421331f10, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc42017a580, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc421364000, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:268 +0xef4

goroutine 328 [semacquire, 2 minutes]:
sync.runtime_notifyListWait(0xc42015c3e8, 0x1)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42015c3d8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc42015c3c0, 0xc420432370, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc420456b00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc420c1af10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc420c1af10, 0x3b9aca00, 0x0, 0x1744b01, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc420c1af10, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc420456b00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc420194dc0, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:269 +0xf44

goroutine 329 [semacquire]:
sync.runtime_notifyListWait(0xc4212c21a8, 0xe)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4212c2198)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc4212c2180, 0xc4205fe260, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc42033cb00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4209cbf10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4209cbf10, 0x3b9aca00, 0x0, 0x1744b01, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4209cbf10, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc42033cb00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc4213640a0, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:270 +0xf94

goroutine 330 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc420458780, 0xc42033cb00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 345 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc420458780, 0xc42017a580)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 75 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc420458780, 0xc420456b00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 76 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc4202be240, 0x1c3faa0, 0xc4213d28a0, 0xc421339c80, 0xc421382cc0, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc4202be240, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc421242240)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc421242240, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc421242240, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 346 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc42132a090, 0x1c3faa0, 0xc421241a10, 0xc421395c80, 0xc42165e900, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc42132a090, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4212dc260)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4212dc260, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4212dc260, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 331 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc4212e6090, 0x1c3faa0, 0xc421aaaf30, 0xc421391c80, 0xc4205c8240, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc4212e6090, 0xc420458780, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4201824c0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4201824c0, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4201824c0, 0x3b9aca00, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 362 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc420494458, 0xc420458780, 0xc42140e3c0, 0xc4202be240, 0xc421382cc0, 0xc420494480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 363 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc421240f60, 0x1c50c20, 0xc4212bba00, 0xc4200a4b40)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:114 +0x6a9
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func2(0xc420558720, 0xc421240f60, 0x1c50c20, 0xc4212bba00, 0xc4200a4b40)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:304 +0x73
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:302 +0x222

goroutine 365 [semacquire]:
sync.runtime_notifyListWait(0xc421c2c028, 0xd)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421c2c018)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop.func1(0x0, 0x0, 0xc420475f00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:506 +0x89
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc421c2c000, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:514 +0xb4
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 364 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc421c2c000, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:532 +0x10c
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 314 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc42043800c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420438000)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4212bb380, 0x1c50c20, 0xc4212bb3c0, 0xc4200a4ae0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:307 +0x47e
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 315 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4203e6000)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc4205736c0, 0x1c50c20, 0xc4212bb3c0, 0xc4204ad900, 0x16, 0xc4203e60f0, 0xc, 0xc4200a4ae0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:143 +0xa4
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 316 [semacquire]:
sync.runtime_SemacquireMutex(0xc420573728, 0xc4212bb300)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420573724)
	/usr/local/go/src/sync/mutex.go:134 +0x108
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc4205736c0, 0x1c50c20, 0xc4212bb3c0)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:159 +0x16a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 353 [semacquire]:
sync.runtime_notifyListWait(0xc4213d80e8, 0xd)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4213d80d8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc4213d80c0, 0xc420084220, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc4202b6000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc421398f10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc421398f10, 0x3b9aca00, 0x0, 0x1744b01, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc421398f10, 0x3b9aca00, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc4202b6000, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc4213da000, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296 +0x470

goroutine 355 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc42128c540, 0xc4202b6000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 356 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc421406000, 0x1c3faa0, 0xc421ada210, 0xc4213b3c80, 0xc42165ede0, 0xc42128c540, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc421406000, 0xc42128c540, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4213d6020)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc4213d6020, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4213d6020, 0x3b9aca00, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 360 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4bc0, 0x72, 0xc4209cd850)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420456898, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420456898, 0xc421536000, 0x8000, 0x8000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420456880, 0xc421536000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420456880, 0xc421536000, 0x8000, 0x8000, 0x8, 0x8, 0x7ff3)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42017e048, 0xc421536000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc4212d0150, 0x7f9a094af9f0, 0xc42017e048, 0x5, 0xc42017e048, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc42044d880, 0x1b6e617, 0xc42044d9a0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc42044d880, 0xc421455000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc421382b40, 0xc420c021f8, 0x9, 0x9, 0x28, 0xc4255ea3f0, 0x0)
	/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x1c36800, 0xc421382b40, 0xc420c021f8, 0x9, 0x9, 0x9, 0xc424584860, 0x43d791, 0xc4213a2f00)
	/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x1c36800, 0xc421382b40, 0xc420c021f8, 0x9, 0x9, 0x1b6e6c8, 0xc4209cdd10, 0x46ebd3)
	/usr/local/go/src/io/io.go:327 +0x58
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.readFrameHeader(0xc420c021f8, 0x9, 0x9, 0x1c36800, 0xc421382b40, 0x0, 0xc400000000, 0xc2b310, 0xc421464528)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420c021c0, 0xc4255ea3f0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*clientConnReadLoop).run(0xc4209cdfb0, 0x1b69e30, 0xc4213e9fb0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1343 +0x8e
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*ClientConn).readLoop(0xc421448000)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1269 +0x76
created by github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*Transport).newClientConn
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:534 +0x651

goroutine 317 [semacquire, 2 minutes]:
sync.runtime_Semacquire(0xc42055872c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc420558720)
	/usr/local/go/src/sync/waitgroup.go:129 +0x72
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run(0xc4212bb9c0, 0x1c50c20, 0xc4212bba00, 0xc4200a4b40)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:307 +0x47e
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:126 +0x116

goroutine 318 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).runProvider(0xc420573730, 0x1c50c20, 0xc4212bba00, 0xc421368320, 0x20, 0xc4203e6ac0, 0xc, 0xc4200a4b40)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:133 +0x187
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:127 +0x190

goroutine 319 [select]:
github.com/prometheus/prometheus/discovery.(*Manager).runUpdater(0xc420573730, 0x1c50c20, 0xc4212bba00)
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:155 +0x13a
created by github.com/prometheus/prometheus/discovery.(*Manager).startProvider
	/go/src/github.com/prometheus/prometheus/discovery/manager.go:128 +0x1cc

goroutine 369 [semacquire]:
sync.runtime_notifyListWait(0xc42015c4a8, 0xe)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc42015c498)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc42015c480, 0xc420432a50, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:438 +0x99
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc420457300)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.processLoop)-fm()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc42132ff10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc42132ff10, 0x3b9aca00, 0x0, 0x1744b01, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc42132ff10, 0x3b9aca00, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc420457300, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x224
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc4200bce60, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:213 +0x334
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296 +0x470

goroutine 370 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc42128c5a0, 0xc420457300)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:103 +0x34
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*controller).Run
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:102 +0x92

goroutine 371 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc4202be2d0, 0x1c3faa0, 0xc421ada0f0, 0xc42141bc80, 0xc420039b00, 0xc42128c5a0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:350 +0x200
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc4202be2d0, 0xc42128c5a0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:321 +0xd49
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil.func1()
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:200 +0x33
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc421242b20)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x54
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc421242b20, 0x3b9aca00, 0x0, 0x8bb2c97001, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xa1
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc421242b20, 0x3b9aca00, 0xc42128c5a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).RunUntil
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:199 +0x172

goroutine 785 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d6520, 0x72, 0xc42444e960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4253f5c18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4253f5c18, 0xc424465000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4253f5c00, 0xc424465000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4253f5c00, 0xc424465000, 0x1000, 0x1000, 0xc423255388, 0xc425446a20, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4244426e8, 0xc424465000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425446a20, 0xc424465000, 0x1000, 0x1000, 0xc42444eb98, 0x403855, 0xc4213470e0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42018f920)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42018f920, 0x1, 0x0, 0x0, 0x0, 0xc424095560, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425446a20)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1340 [runnable]:
net/http.(*connReader).backgroundRead(0xc42512dd10)
	/usr/local/go/src/net/http/server.go:667
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:664 +0xce

goroutine 336 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc42017e060, 0xc420458720, 0xc420459380, 0xc42132a000, 0xc4214e2120, 0xc42017e068)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 1178 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171da0, 0x72, 0xc4241b9960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425e31218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425e31218, 0xc424029000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425e31200, 0xc424029000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425e31200, 0xc424029000, 0x1000, 0x1000, 0xc422f501d0, 0xc425d48b40, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423255250, 0xc424029000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425d48b40, 0xc424029000, 0x1000, 0x1000, 0xc4241b9b98, 0x403855, 0xc426fc5a40)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f7440)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f7440, 0x1, 0x0, 0x0, 0x0, 0xc426fc5980, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d48b40)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1206 [chan receive]:
github.com/prometheus/prometheus/scrape.(*scrapeLoop).stop(0xc42204f800)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:781 +0x46
github.com/prometheus/prometheus/scrape.(*scrapePool).sync.func1(0xc421cfe340, 0x1c3da60, 0xc42204f800)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:318 +0x31
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:317 +0x5b4

goroutine 1187 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171e70, 0x72, 0xc423549960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425e30f98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425e30f98, 0xc423f6d000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425e30f80, 0xc423f6d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425e30f80, 0xc423f6d000, 0x1000, 0x1000, 0xc423255240, 0xc425d486c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426dcd2a8, 0xc423f6d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425d486c0, 0xc423f6d000, 0x1000, 0x1000, 0xc423549b98, 0x403855, 0xc424014060)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421245ec0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421245ec0, 0x1, 0x0, 0x0, 0x0, 0xc426453f80, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d486c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1182 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ce0, 0xc42228e5d0, 0x8000000000000000, 0x7fffffffffffffff, 0x1c4f0a0, 0xc421b4d260, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ce0, 0xc42228e5d0, 0x8000000000000000, 0x7fffffffffffffff, 0x8, 0xc420392120, 0x2874a30, 0xc4202d0240)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/web/api/v1.(*API).labelValues(0xc420573ab0, 0xc4255e3800, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:341 +0x142
github.com/prometheus/prometheus/web/api/v1.(*API).(github.com/prometheus/prometheus/web/api/v1.labelValues)-fm(0xc4255e3800, 0xc42896c120, 0xc426d54690, 0x3)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:179 +0x34
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1c45ca0, 0xc42896c120, 0xc4255e3800)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:159 +0x55
net/http.HandlerFunc.ServeHTTP(0xc4202ff110, 0x1c45ca0, 0xc42896c120, 0xc4255e3800)
	/usr/local/go/src/net/http/server.go:1947 +0x44
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1c3b7e0, 0xc4202ff110, 0x1c474e0, 0xc426d54690, 0xc4255e3800)
	/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x7c
net/http.(Handler).ServeHTTP-fm(0x1c474e0, 0xc426d54690, 0xc4255e3800)
	/usr/local/go/src/net/http/h2_bundle.go:5475 +0x4d
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc42896c0e0, 0xc4255e3800)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc42896c0e0, 0xc4255e3800)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc42896c0e0, 0xc4255e3700, 0xc42896c100, 0x1, 0x1)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc4206f0540, 0x1c47420, 0xc42896c0e0, 0xc4255e3700)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc420126740, 0x1c47420, 0xc42896c0e0, 0xc4255e3700)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.StripPrefix.func1(0x1c47420, 0xc42896c0e0, 0xc4255e3600)
	/usr/local/go/src/net/http/server.go:1986 +0x19a
net/http.HandlerFunc.ServeHTTP(0xc420f1e2d0, 0x1c47420, 0xc42896c0e0, 0xc4255e3600)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc42896c0e0, 0xc4255e3600)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc424e3fdc0, 0xc4255e3400)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc424e3fdc0, 0xc4255e3400)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc424e3fdc0, 0xc4255e3400)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc425fe32c0, 0x1c50c20, 0xc421c67dc0)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1205 [select]:
net/http.(*persistConn).writeLoop(0xc425d1b560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1087 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119768, 0x72, 0xc42764e960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f63598, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f63598, 0xc425194000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f63580, 0xc425194000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f63580, 0xc425194000, 0x1000, 0x1000, 0xc42773e058, 0xc425fc0c60, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426de40b0, 0xc425194000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425fc0c60, 0xc425194000, 0x1000, 0x1000, 0xc42764eb98, 0x403855, 0xc4261eb260)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4203ca720)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4203ca720, 0x1, 0x0, 0x0, 0x0, 0xc421299c20, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425fc0c60)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 864 [select]:
net/http.(*persistConn).writeLoop(0xc425fccb40)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1176 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171f40, 0x72, 0xc42354b960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425e30b98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425e30b98, 0xc423fa0000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425e30b80, 0xc423fa0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425e30b80, 0xc423fa0000, 0x1000, 0x1000, 0xc422f50150, 0xc425d4fc20, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423255230, 0xc423fa0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425d4fc20, 0xc423fa0000, 0x1000, 0x1000, 0xc42354bb98, 0x403855, 0xc426fc5920)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f72c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f72c0, 0x1, 0x0, 0x0, 0x0, 0xc426fc5860, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d4fc20)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1188 [select]:
net/http.(*persistConn).writeLoop(0xc425d486c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 847 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091199d8, 0x72, 0xc42764f960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425ff3518, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425ff3518, 0xc4261ac000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425ff3500, 0xc4261ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425ff3500, 0xc4261ac000, 0x1000, 0x1000, 0xc4232550e8, 0xc425ffd7a0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4232547a0, 0xc4261ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425ffd7a0, 0xc4261ac000, 0x1000, 0x1000, 0xc42764fb98, 0x403855, 0xc4261ea540)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c4e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c4e0, 0x1, 0x0, 0x0, 0x0, 0xc4261ea480, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425ffd7a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 858 [select]:
net/http.(*persistConn).writeLoop(0xc425ff4000)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 863 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119908, 0x72, 0xc4215a5960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425fd3518, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425fd3518, 0xc420fe4000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425fd3500, 0xc420fe4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425fd3500, 0xc420fe4000, 0x1000, 0x1000, 0xc425511e08, 0xc425fccb40, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425511e00, 0xc420fe4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425fccb40, 0xc420fe4000, 0x1000, 0x1000, 0xc4215a5b98, 0x403855, 0xc421299bc0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc420413e00)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc420413e00, 0x1, 0x0, 0x0, 0x0, 0xc421299b00, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425fccb40)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 652 [select]:
net/http.(*persistConn).writeLoop(0xc425446900)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 452 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc42017e098, 0xc420458780, 0xc421346060, 0xc42132a090, 0xc42165e900, 0xc42017e0a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 642 [semacquire, 2 minutes]:
sync.runtime_notifyListWait(0xc4200ea4f8, 0x0)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4200ea4e8)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop.func1(0x0, 0x0, 0xc4213ea700)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:506 +0x89
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc4200ea4d0, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:514 +0xb4
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 552 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc424ad27e0, 0x1620a73d15c, 0x1620a78653c, 0x1c4f0a0, 0xc421896380, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc424ad27e0, 0x1620a73d15c, 0x1620a78653c, 0xffffffc91d4b2b09, 0x2883180, 0xac69d7c27932f58, 0xc427932f00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc424ad27e0, 0x7f9a0946e1d8, 0xc42016a380, 0xc423c1b900, 0x40edfd, 0x18947e0, 0x18848a0, 0xc427932f01)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc424ad27e0, 0xc423c1b950, 0xc423c1b900, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc424ad27e0, 0xc423c1b950, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc423c1b950, 0x1c50c60, 0xc420044018, 0xc42707ff22)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc42707ff22, 0xe, 0xbea0b9094ac39aef, 0xef6afe309, 0x2883180, 0x3639313962366330, 0x3638353637633534, 0x6263396363326532, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*AlertingRule).Eval(0xc420610500, 0x1c50c60, 0xc420044018, 0xbea0b9094ac39aef, 0xef6afe309, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/alerting.go:177 +0x102
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b9094ac39aef, 0xef6afe309, 0x2883180, 0xc4202be510, 0x0, 0x1c59bc0, 0xc420610500)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be510, 0x1c50c60, 0xc420044018, 0xbea0b9094ac39aef, 0xef6afe309, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be510, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:230 +0x257
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be510)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 451 [semacquire, 2 minutes]:
sync.runtime_notifyListWait(0xc4214647c0, 0xc400000000)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4214647b0)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc4214647a8, 0xc4215e8400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421464780, 0xc4215e8400, 0x200, 0x200, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc4204501e0, 0xc42129100d, 0x0)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc4204501e0, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc4204501e0, 0x18542c0, 0xc4212dc2c0, 0x2, 0x3f3)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc4213d2870, 0xc4213f0000, 0x400, 0x400, 0xc4213de300, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc4213ce3c0, 0x0, 0x1c37c80, 0xc4213de300, 0x403855, 0xc42128c960, 0xc421311760, 0x1adaf01, 0x14dd26f)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc4212dc240, 0x1b6a8a8, 0x0, 0x0, 0x0, 0x2, 0xc421311780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc4213d28a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 750 [select]:
net/http.(*persistConn).writeLoop(0xc426ca65a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 643 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a42d0, 0x72, 0xc423da0960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc423866298, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc423866298, 0xc42166e000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc423866280, 0xc42166e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc423866280, 0xc42166e000, 0x1000, 0x1000, 0xc422f504a0, 0xc4247a6000, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4247a04f0, 0xc42166e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4247a6000, 0xc42166e000, 0x1000, 0x1000, 0xc423da0b98, 0x403855, 0xc42127a240)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c300)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c300, 0x1, 0x0, 0x0, 0x0, 0xc423e7f080, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4247a6000)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 646 [select]:
net/http.(*persistConn).writeLoop(0xc424454000)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 738 [select]:
net/http.(*persistConn).writeLoop(0xc421f38240)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 737 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4880, 0x72, 0xc42188c960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc421f3d098, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc421f3d098, 0xc420fb3000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc421f3d080, 0xc420fb3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc421f3d080, 0xc420fb3000, 0x1000, 0x1000, 0xc4232552f0, 0xc421f38240, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42017e178, 0xc420fb3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc421f38240, 0xc420fb3000, 0x1000, 0x1000, 0xc42188cb98, 0x403855, 0xc421cee3c0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c2a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c2a0, 0x1, 0x0, 0x0, 0x0, 0xc421cc7c20, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc421f38240)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 740 [select]:
net/http.(*persistConn).writeLoop(0xc4218f4480)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 458 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a46e0, 0x72, 0xc421c3a960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4221acb98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4221acb98, 0xc422318000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4221acb80, 0xc422318000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4221acb80, 0xc422318000, 0x1000, 0x1000, 0xc422f503a8, 0xc421a41200, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc421eb2428, 0xc422318000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc421a41200, 0xc422318000, 0x1000, 0x1000, 0xc421c3ab98, 0x403855, 0xc4212997a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421245740)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421245740, 0x1, 0x0, 0x0, 0x0, 0xc4240952c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc421a41200)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 875 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119b78, 0x72, 0xc427dab960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261d7d18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261d7d18, 0xc42617a000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261d7d00, 0xc42617a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261d7d00, 0xc42617a000, 0x1000, 0x1000, 0xc425511408, 0xc4261166c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbb928, 0xc42617a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4261166c0, 0xc42617a000, 0x1000, 0x1000, 0xc427dabb98, 0x403855, 0xc421eff4a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421da9f80)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421da9f80, 0x1, 0x0, 0x0, 0x0, 0xc421eff3e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4261166c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 368 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc421f5acc0, 0x6, 0x6, 0x1620a77fc17, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421b3c1c0, 0xc421f5acc0, 0x6, 0x6, 0x447, 0x1620a77fc17, 0x3ff0000000000000, 0xc4275ce240, 0xc423dedf10)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4251f8060, 0xc421f5acc0, 0x6, 0x6, 0x447, 0x1620a77fc17, 0x3ff0000000000000, 0xc420084060, 0x7005)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420275400, 0xc4266b0000, 0x1af005, 0x30f489, 0xbea0b9028fb3f630, 0x8b24cf08c, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420275400, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 393 [semacquire]:
sync.runtime_notifyListWait(0xc421a556c0, 0xc40000000c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421a556b0)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc421a556a8, 0xc427788001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421a55680, 0xc427788001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc421363680, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc421363680, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc421363680, 0x18542c0, 0xc428f252c0, 0x1c38800, 0xc4283c8f00)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc421aaaf00, 0xc427792000, 0x2000, 0x2600, 0xc4247979c0, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc42146e500, 0x0, 0x1c37c80, 0xc4247979c0, 0x1a80, 0x1c38800, 0xc421b9a238, 0x454dc0, 0xc421b9a1e0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421bab160, 0xc4215cdfa8, 0x7, 0x1c38800, 0xc4283c8f00, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc421aaaf30)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 394 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc421240210, 0x1c50c20, 0xc420047900, 0xc4200a4960)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:114 +0x6a9
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func2(0xc420558030, 0xc421240210, 0x1c50c20, 0xc420047900, 0xc4200a4960)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:304 +0x73
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:302 +0x222

goroutine 390 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc42039e000, 0xc42128c2a0, 0xc420459aa0, 0xc4212e6000, 0xc4205c8060, 0xc42039e020)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 389 [semacquire]:
sync.runtime_notifyListWait(0xc421344400, 0xc40000003c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4213443f0)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc4213443e8, 0xc42144a001, 0x7dff, 0x7dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc4213443c0, 0xc42144a001, 0x7dff, 0x7dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc4213621e0, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc4213621e0, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc4213621e0, 0x18542c0, 0xc4292dbbe0, 0x1c38700, 0xc42918cb00)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc4202c6660, 0xc421bb6000, 0x8000, 0xa000, 0xc4291dfc40, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc420174410, 0x0, 0x1c37c80, 0xc4291dfc40, 0x5000, 0x1c38700, 0xc42128c0b8, 0x454dc0, 0xc42128c060)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc42060e060, 0xc4215cbfa8, 0x8, 0x1c38700, 0xc42918cb00, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc420393d70)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 651 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4060, 0x72, 0xc423da6960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4253f5998, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4253f5998, 0xc425452000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4253f5980, 0xc425452000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4253f5980, 0xc425452000, 0x1000, 0x1000, 0xc423255358, 0xc425446900, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425433018, 0xc425452000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425446900, 0xc425452000, 0x1000, 0x1000, 0xc423da6b98, 0x403855, 0xc4213467e0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42165f680)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42165f680, 0x1, 0x0, 0x0, 0x0, 0xc424014f00, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425446900)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 641 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc4200ea4d0, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:532 +0x10c
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 644 [select]:
net/http.(*persistConn).writeLoop(0xc4247a6000)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 551 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc4214e3e60, 0x1620a73aa36, 0x1620a783e16, 0x1c4f0a0, 0xc421cebc10, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc4214e3e60, 0x1620a73aa36, 0x1620a783e16, 0xffffffc6c7ed6f8e, 0x2883180, 0x975131827931178, 0xc427931120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc4214e3e60, 0x7f9a0946e1d8, 0xc42016a380, 0xc4221cf6d0, 0xc425fdc428, 0x14, 0xb, 0xc427931248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc4214e3e60, 0xc4221cf720, 0xc4221cf6d0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc4214e3e60, 0xc4221cf720, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4221cf720, 0x1c50c60, 0xc420044018, 0xc428258000)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc428258000, 0x5c, 0xbea0b906c971c368, 0xca152278e, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x6f725f626f6aff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc42017d860, 0x1c50c60, 0xc420044018, 0xbea0b906c971c368, 0xca152278e, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b906c971c368, 0xca152278e, 0x2883180, 0xc42132a2d0, 0x0, 0x1c59c20, 0xc42017d860)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a2d0, 0x1c50c60, 0xc420044018, 0xbea0b906c971c368, 0xca152278e, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a2d0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a2d0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 553 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc421245080, 0x1620a734556, 0x1620a77d936, 0x1c4f0a0, 0xc421ca83f0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc421245080, 0x1620a734556, 0x1620a77d936, 0xffffffc0c4b8f419, 0x2883180, 0x13f8fe6127809178, 0xc427809120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc421245080, 0x7f9a0946e1d8, 0xc42016a380, 0xc42229c640, 0xc421d928bf, 0xc4215d1c80, 0xc420412ae0, 0xc4215d1d08)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc421245080, 0xc42229c690, 0xc42229c640, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc421245080, 0xc42229c690, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc42229c690, 0x1c50c60, 0xc420044018, 0xc421d928a0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc421d928a0, 0x21, 0xbea0b90053f5cbf1, 0x69e1dac19, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x722e65646f6eff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206cf9f0, 0x1c50c60, 0xc420044018, 0xbea0b90053f5cbf1, 0x69e1dac19, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b90053f5cbf1, 0x69e1dac19, 0x2883180, 0xc4202be6c0, 0x0, 0x1c59c20, 0xc4206cf9f0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be6c0, 0x1c50c60, 0xc420044018, 0xbea0b90053f5cbf1, 0x69e1dac19, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be6c0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be6c0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 391 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc42039e0b0, 0xc420458780, 0xc421b9a060, 0xc4212e6090, 0xc4205c8240, 0xc42039e0c0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 555 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc426c280c0, 0x1620a73af5c, 0x1620a78433c, 0x1c4f0a0, 0xc421cebe40, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc426c280c0, 0x1620a73af5c, 0x1620a78433c, 0xffffffc7168172e6, 0x2883180, 0x1c70a3a4282b8f58, 0xc4282b8f00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc426c280c0, 0x7f9a0946e1d8, 0xc42016a380, 0xc4221cf9a0, 0x40edfd, 0x18947e0, 0x18848a0, 0xc42047ae01)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc426c280c0, 0xc4221cf9f0, 0xc4221cf9a0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc426c280c0, 0xc4221cf9f0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4221cf9f0, 0x1c50c60, 0xc420044018, 0xc428258780)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc428258780, 0x5f, 0xbea0b9071c6afcc4, 0xcefe62ae6, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*AlertingRule).Eval(0xc420610700, 0x1c50c60, 0xc420044018, 0xbea0b9071c6afcc4, 0xcefe62ae6, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/alerting.go:177 +0x102
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b9071c6afcc4, 0xcefe62ae6, 0x2883180, 0xc4202be5a0, 0x0, 0x1c59bc0, 0xc420610700)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be5a0, 0x1c50c60, 0xc420044018, 0xbea0b9071c6afcc4, 0xcefe62ae6, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be5a0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be5a0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 579 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc421da9200, 0x1620a6a09ae, 0x1620a77c54e, 0x1c4f0a0, 0xc421c88480, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc421da9200, 0x1620a6a09ae, 0x1620a77c54e, 0xffffff33e2301446, 0x2883180, 0xe3e9ffb215c8f58, 0xc4215c8f00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc421da9200, 0x7f9a0946e1d8, 0xc42016a380, 0xc42616e320, 0x40edfd, 0x18947e0, 0x18848a0, 0xc421eaa401)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc421da9200, 0xc42616e370, 0xc42616e320, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc421da9200, 0xc42616e370, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc42616e370, 0x1c50c60, 0xc420044018, 0xc421d477c0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc421d477c0, 0x31, 0xbea0b8ff0e3c4e3e, 0x56e5e3c46, 0x2883180, 0x376, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*AlertingRule).Eval(0xc420457f00, 0x1c50c60, 0xc420044018, 0xbea0b8ff0e3c4e3e, 0x56e5e3c46, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/alerting.go:177 +0x102
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b8ff0e3c4e3e, 0x56e5e3c46, 0x2883180, 0xc4202be3f0, 0x0, 0x1c59bc0, 0xc420457f00)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be3f0, 0x1c50c60, 0xc420044018, 0xbea0b8ff0e3c4e3e, 0x56e5e3c46, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be3f0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be3f0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 580 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc4214e2b40, 0x1620a736d1b, 0x1620a7800fb, 0x1c4f0a0, 0xc421cea9f0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc4214e2b40, 0x1620a736d1b, 0x1620a7800fb, 0xffffffc323875bc8, 0x2883180, 0x1ebba18a27737178, 0xc427737120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc4214e2b40, 0x7f9a0946e1d8, 0xc42016a380, 0xc4221ceb40, 0xc425ff81e8, 0x14, 0xb, 0xc427737248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc4214e2b40, 0xc4221ceb90, 0xc4221ceb40, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc4214e2b40, 0xc4221ceb90, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4221ceb90, 0x1c50c60, 0xc420044018, 0xc427813ab0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc427813ab0, 0x6d, 0xbea0b902deb84fa2, 0x8fcec13c8, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x656d5f626f6aff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206cebe0, 0x1c50c60, 0xc420044018, 0xbea0b902deb84fa2, 0x8fcec13c8, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b902deb84fa2, 0x8fcec13c8, 0x2883180, 0xc42132a510, 0x0, 0x1c59c20, 0xc4206cebe0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a510, 0x1c50c60, 0xc420044018, 0xbea0b902deb84fa2, 0x8fcec13c8, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a510, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a510)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 578 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc424ad2960, 0x1620a73e218, 0x1620a7875f8, 0x1c4f0a0, 0xc4218966d0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc424ad2960, 0x1620a73e218, 0x1620a7875f8, 0xffffffca1ca866a0, 0x2883180, 0x1bb945db2773b178, 0xc42773b120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc424ad2960, 0x7f9a0946e1d8, 0xc42016a380, 0xc423c1bb80, 0xc425fdc4e8, 0x14, 0xb, 0xc42773b248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc424ad2960, 0xc423c1bbd0, 0xc423c1bb80, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc424ad2960, 0xc423c1bbd0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc423c1bbd0, 0x1c50c60, 0xc420044018, 0xc427221320)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc427221320, 0x5c, 0xbea0b90a5bb5ae8a, 0xff60d1ea0, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206ce640, 0x1c50c60, 0xc420044018, 0xbea0b90a5bb5ae8a, 0xff60d1ea0, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b90a5bb5ae8a, 0xff60d1ea0, 0x2883180, 0xc42132a3f0, 0x0, 0x1c59c20, 0xc4206ce640)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a3f0, 0x1c50c60, 0xc420044018, 0xbea0b90a5bb5ae8a, 0xff60d1ea0, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a3f0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:230 +0x257
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a3f0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 557 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc42136cfc0, 0x1620a73b1fe, 0x1620a7845de, 0x1c4f0a0, 0xc42049d330, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc42136cfc0, 0x1620a73b1fe, 0x1620a7845de, 0xffffffc73ea66f35, 0x2883180, 0x8f963d7271cf178, 0xc4271cf120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc42136cfc0, 0x7f9a0946e1d8, 0xc42016a380, 0xc42240bef0, 0xc425fdc4e8, 0x14, 0xb, 0xc4271cf248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc42136cfc0, 0xc42240bf40, 0xc42240bef0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc42136cfc0, 0xc42240bf40, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc42240bf40, 0x1c50c60, 0xc420044018, 0xc427244f50)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc427244f50, 0x70, 0xbea0b90748f52f0f, 0xd180b2735, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x706f5f626f6aff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206cef50, 0x1c50c60, 0xc420044018, 0xbea0b90748f52f0f, 0xd180b2735, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b90748f52f0f, 0xd180b2735, 0x2883180, 0xc42132a5a0, 0x0, 0x1c59c20, 0xc4206cef50)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a5a0, 0x1c50c60, 0xc420044018, 0xbea0b90748f52f0f, 0xd180b2735, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a5a0, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a5a0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 556 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc4265d0b40, 0x1620a73bb43, 0x1620a784f23, 0x1c4f0a0, 0xc420fe8420, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc4265d0b40, 0x1620a73bb43, 0x1620a784f23, 0xffffffc7cc15721b, 0x2883180, 0x1f32898c282c7178, 0xc4282c7120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc4265d0b40, 0x7f9a0946e1d8, 0xc42016a380, 0xc4271b75e0, 0xc425fbdaa8, 0x14, 0xb, 0xc4282c7248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc4265d0b40, 0xc4271b7630, 0xc4271b75e0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc4265d0b40, 0xc4271b7630, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4271b7630, 0x1c50c60, 0xc420044018, 0xc42827f080)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc42827f080, 0x55, 0xbea0b907df2e9e0e, 0xda57a2a1b, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x65723a626f6aff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc42017d180, 0x1c50c60, 0xc420044018, 0xbea0b907df2e9e0e, 0xda57a2a1b, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b907df2e9e0e, 0xda57a2a1b, 0x2883180, 0xc4202be630, 0x0, 0x1c59c20, 0xc42017d180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be630, 0x1c50c60, 0xc420044018, 0xbea0b907df2e9e0e, 0xda57a2a1b, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be630, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be630)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 560 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc4200ea420, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 554 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc426c28360, 0x1620a73e483, 0x1620a787863, 0x1c4f0a0, 0xc420e18250, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc426c28360, 0x1620a73e483, 0x1620a787863, 0xffffffca4185b96a, 0x2883180, 0x4fb8aa3271cb178, 0xc4271cb120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc426c28360, 0x7f9a0946e1d8, 0xc42016a380, 0xc4221cfdb0, 0xc425ee96e8, 0x14, 0xb, 0xc4271cb248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc426c28360, 0xc4221cfe00, 0xc4221cfdb0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc426c28360, 0xc4221cfe00, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4221cfe00, 0x1c50c60, 0xc420044018, 0xc428289b90)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc428289b90, 0x63, 0xbea0b90a84f8374b, 0x101aea716a, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206ce8c0, 0x1c50c60, 0xc420044018, 0xbea0b90a84f8374b, 0x101aea716a, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b90a84f8374b, 0x101aea716a, 0x2883180, 0xc42132a480, 0x0, 0x1c59c20, 0xc4206ce8c0)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a480, 0x1c50c60, 0xc420044018, 0xbea0b90a84f8374b, 0x101aea716a, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a480, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:230 +0x257
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a480)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 588 [semacquire]:
sync.runtime_notifyListWait(0xc421ccd080, 0xc40000000d)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421ccd070)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc421ccd068, 0xc4277ac001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421ccd040, 0xc4277ac001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc4203c5c20, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc4203c5c20, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc4203c5c20, 0x18542c0, 0xc428f195e0, 0x1c38800, 0xc428ebf500)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc420e31620, 0xc4277ba000, 0x2000, 0x2600, 0xc428ed8ac0, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc42135cb40, 0x0, 0x1c37c80, 0xc428ed8ac0, 0x1a80, 0x1c38800, 0xc421299af8, 0x454dc0, 0xc421299aa0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421e1f080, 0xc4213b5fa8, 0x7, 0x1c38800, 0xc428ebf500, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc420e31650)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 581 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc42122d560, 0x1620a735d39, 0x1620a77f119, 0x1c4f0a0, 0xc421bd55d0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc42122d560, 0x1620a735d39, 0x1620a77f119, 0xffffffc2312eeff7, 0x2883180, 0x1acdd6b927935178, 0xc427935120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc42122d560, 0x7f9a0946e1d8, 0xc42016a380, 0xc42403a050, 0xc421913560, 0xc4213a2180, 0xc420f58e40, 0x17)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc42122d560, 0xc42403a0a0, 0xc42403a050, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc42122d560, 0xc42403a0a0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc42403a0a0, 0x1c50c60, 0xc420044018, 0xc4216a3260)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc4216a3260, 0x61, 0xbea0b901dacb0bd9, 0x80a93a7f7, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x75722e73386bff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206cf270, 0x1c50c60, 0xc420044018, 0xbea0b901dacb0bd9, 0x80a93a7f7, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b901dacb0bd9, 0x80a93a7f7, 0x2883180, 0xc42132a630, 0x0, 0x1c59c20, 0xc4206cf270)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a630, 0x1c50c60, 0xc420044018, 0xbea0b901dacb0bd9, 0x80a93a7f7, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a630, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a630)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 582 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc42136d2c0, 0x1620a73b4b2, 0x1620a784892, 0x1c4f0a0, 0xc42049d5f0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc42136d2c0, 0x1620a73b4b2, 0x1620a784892, 0xffffffc767eecd5f, 0x2883180, 0x3243fb6a2558cf58, 0xc42558cf00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc42136d2c0, 0x7f9a0946e1d8, 0xc42016a380, 0xc4282bc2d0, 0x40edfd, 0x18947e0, 0x18848a0, 0x1ade901)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc42136d2c0, 0xc4282bc320, 0xc4282bc2d0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc42136d2c0, 0xc4282bc320, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4282bc320, 0x1c50c60, 0xc420044018, 0xc42645cd20)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc42645cd20, 0xe2, 0xbea0b907723d8d35, 0xd4153855f, 0x2883180, 0x3634653432333234, 0x3835613533343636, 0x3733373736336136, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*AlertingRule).Eval(0xc420457b80, 0x1c50c60, 0xc420044018, 0xbea0b907723d8d35, 0xd4153855f, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/alerting.go:177 +0x102
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b907723d8d35, 0xd4153855f, 0x2883180, 0xc4202be360, 0x0, 0x1c59bc0, 0xc420457b80)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be360, 0x1c50c60, 0xc420044018, 0xbea0b907723d8d35, 0xd4153855f, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be360, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be360)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 558 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run(0xc420272150, 0x1c50c20, 0xc4212bad40, 0xc4200a4a80)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:156 +0x624
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func1(0xc42016e1a0, 0xc420272150, 0x1c50c20, 0xc4212bad40, 0xc4200a4a80)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:284 +0x73
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:282 +0xa93

goroutine 577 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc4265d09c0, 0x1620a73a81a, 0x1620a783bfa, 0x1c4f0a0, 0xc4205aff40, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc4265d09c0, 0x1620a73a81a, 0x1620a783bfa, 0xffffffc6a7b79503, 0x2883180, 0x24da2ebe2792f178, 0xc42792f120)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc4265d09c0, 0x7f9a0946e1d8, 0xc42016a380, 0xc4271b7360, 0xc425e54068, 0x14, 0x20, 0xc42792f248)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc4265d09c0, 0xc4271b73b0, 0xc4271b7360, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc4265d09c0, 0xc4271b73b0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc4271b73b0, 0x1c50c60, 0xc420044018, 0xc427171e00)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc427171e00, 0x60, 0xbea0b906a4d6b2f8, 0xc811c4d03, 0x2883180, 0x722e676e6964726f, 0x6d616eff73656c75, 0x75613a626f6aff65, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*RecordingRule).Eval(0xc4206ce370, 0x1c50c60, 0xc420044018, 0xbea0b906a4d6b2f8, 0xc811c4d03, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/recording.go:57 +0xcf
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b906a4d6b2f8, 0xc811c4d03, 0x2883180, 0xc42132a360, 0x0, 0x1c59c20, 0xc4206ce370)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc42132a360, 0x1c50c60, 0xc420044018, 0xbea0b906a4d6b2f8, 0xc811c4d03, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc42132a360, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc42132a360)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 576 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc42122c3c0, 0x1620a7343c3, 0x1620a77d7a3, 0x1c4f0a0, 0xc421d0e480, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc42122c3c0, 0x1620a7343c3, 0x1620a77d7a3, 0xffffffc0acafe49e, 0x2883180, 0x37897f4c2773cf58, 0xc42773cf00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc42122c3c0, 0x7f9a0946e1d8, 0xc42016a380, 0xc425b9c8c0, 0x40edfd, 0x18947e0, 0x18848a0, 0xc42773cf01)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc42122c3c0, 0xc425b9c910, 0xc425b9c8c0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc42122c3c0, 0xc425b9c910, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc425b9c910, 0x1c50c60, 0xc420044018, 0xc42196c201)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/rules.EngineQueryFunc.func1(0x1c50c60, 0xc420044018, 0xc42196c201, 0x7, 0xbea0b90037878685, 0x686149c9e, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:115 +0xc5
github.com/prometheus/prometheus/rules.(*AlertingRule).Eval(0xc420610280, 0x1c50c60, 0xc420044018, 0xbea0b90037878685, 0x686149c9e, 0x2883180, 0xc420126300, 0xc420530480, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/rules/alerting.go:177 +0x102
github.com/prometheus/prometheus/rules.(*Group).Eval.func1(0x1c50c60, 0xc420044018, 0xbea0b90037878685, 0x686149c9e, 0x2883180, 0xc4202be480, 0x0, 0x1c59bc0, 0xc420610280)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:334 +0x139
github.com/prometheus/prometheus/rules.(*Group).Eval(0xc4202be480, 0x1c50c60, 0xc420044018, 0xbea0b90037878685, 0x686149c9e, 0x2883180)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:402 +0xad
github.com/prometheus/prometheus/rules.(*Group).run.func1()
	/go/src/github.com/prometheus/prometheus/rules/manager.go:204 +0xa7
github.com/prometheus/prometheus/rules.(*Group).run(0xc4202be480, 0x1c50c60, 0xc420044018)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:210 +0x1d4
github.com/prometheus/prometheus/rules.(*Manager).Update.func1.1(0xc42016a940, 0xc4202be480)
	/go/src/github.com/prometheus/prometheus/rules/manager.go:505 +0x60
created by github.com/prometheus/prometheus/rules.(*Manager).Update.func1
	/go/src/github.com/prometheus/prometheus/rules/manager.go:500 +0x56

goroutine 584 [semacquire]:
sync.runtime_notifyListWait(0xc421464540, 0xc400000088)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421464530)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc421464528, 0xc421445801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421464500, 0xc421445801, 0x5ff, 0x5ff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc4203c43c0, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc4203c43c0, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc4203c43c0, 0x18542c0, 0xc4292e9420, 0x1c385e0, 0xc4237aec60)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc4212419e0, 0xc4212d6000, 0x400, 0x400, 0xc4292b98c0, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc4203cd270, 0x0, 0x1c37c80, 0xc4292b98c0, 0x240, 0x1c385e0, 0xc421299018, 0x454dc0, 0xc421298fc0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc420427660, 0xc4215cffa8, 0x8, 0x1c385e0, 0xc4237aec60, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc421241a10)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 585 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc420494580, 0xc42128c5a0, 0xc421299440, 0xc4202be2d0, 0xc420039b00, 0xc420494588)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 586 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc42000c0b8, 0xc420458480, 0xc421299920, 0xc420262a20, 0xc421c5c060, 0xc42000c0e0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 461 [select]:
net/http.(*persistConn).writeLoop(0xc42176a900)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 590 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc421cf00b0, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 589 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1(0xc4241d93e0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:75 +0x2f4
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func4(0x1ab44a0, 0xc423e47800, 0x1ab44a0, 0xc4275ecf00)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:109 +0x274
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc421eb6700, 0xc421eb6740, 0xc421eb6720, 0x1ab44a0, 0xc423e47800, 0x1ab44a0, 0xc4275ecf00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc421cf00b0, 0xc42128c000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 413 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run.func1(0xc422f7b2f0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:66 +0x134
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run.func4(0x1ab3ba0, 0xc424e2a840, 0x1ab3ba0, 0xc424190b00)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:100 +0x273
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc4213500c0, 0xc421350180, 0xc421350100, 0x1ab3ba0, 0xc424e2a840, 0x1ab3ba0, 0xc424190b00)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc4200ea2c0, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 414 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc4200ea2c0, 0xc42128c2a0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 559 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1(0xc4252fd950)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:81 +0x2f3
github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func3(0x1ab2fa0, 0xc4259e8c60, 0x1ab2fa0, 0xc425934360)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:106 +0x273
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc421b26760, 0xc421b26780, 0xc421b267a0, 0x1ab2fa0, 0xc4259e8c60, 0x1ab2fa0, 0xc425934360)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc4200ea420, 0xc420458780)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 416 [semacquire]:
sync.runtime_notifyListWait(0xc4203a6040, 0xc40000003f)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4203a6030)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc4203a6028, 0xc421432001, 0x7dff, 0x7dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc4203a6000, 0xc421432001, 0x7dff, 0x7dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc4212c4000, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc4212c4000, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc4212c4000, 0x18542c0, 0xc4292e8ea0, 0x1c38700, 0xc429149600)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc4213084b0, 0xc421582000, 0x8000, 0xa000, 0xc4292b9880, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc4212f6050, 0x0, 0x1c37c80, 0xc4292b9880, 0x5000, 0x1c38700, 0xc42127b2b8, 0x454dc0, 0xc42127b260)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421350300, 0xc42129ffa8, 0x8, 0x1c38700, 0xc429149600, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc4213084e0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 453 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run.func1(0xc422dbe4e0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:66 +0x134
github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run.func4(0x1ab3ba0, 0xc422ff2b00, 0x1ab3ba0, 0xc423db3340)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:100 +0x273
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc4212c8960, 0xc4212c89a0, 0xc4212c8980, 0x1ab3ba0, 0xc422ff2b00, 0x1ab3ba0, 0xc423db3340)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc4201a3810, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 454 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc4201a3810, 0xc420458720)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 455 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc4213e2008, 0xc42128c540, 0xc4212d2ae0, 0xc421406000, 0xc42165ede0, 0xc4213e2010)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 594 [semacquire]:
sync.runtime_notifyListWait(0xc421345580, 0xc40000000c)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc421345570)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc421345568, 0xc4277ca001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc421345540, 0xc4277ca001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc421ae8000, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc421ae8000, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc421ae8000, 0x18542c0, 0xc428f123c0, 0x1c38800, 0xc428ef4c00)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc421ada0c0, 0xc4277d6000, 0x2000, 0x2600, 0xc428eab440, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc4214c80a0, 0x0, 0x1c37c80, 0xc428eab440, 0x1a80, 0x1c38800, 0xc421aec058, 0x454dc0, 0xc421aec000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421880e20, 0xc4213b1fa8, 0x7, 0x1c38800, 0xc428ef4c00, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc421ada0f0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 596 [semacquire]:
sync.runtime_notifyListWait(0xc4219e5580, 0xc40000000a)
	/usr/local/go/src/runtime/sema.go:510 +0x10b
sync.(*Cond).Wait(0xc4219e5570)
	/usr/local/go/src/sync/cond.go:56 +0x80
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.(*pipe).Read(0xc4219e5568, 0xc4277ae001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:64 +0x8f
github.com/prometheus/prometheus/vendor/golang.org/x/net/http2.transportResponseBody.Read(0xc4219e5540, 0xc4277ae001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1588 +0xa1
encoding/json.(*Decoder).refill(0xc421ae81e0, 0xc6a80a, 0x9)
	/usr/local/go/src/encoding/json/stream.go:159 +0x132
encoding/json.(*Decoder).readValue(0xc421ae81e0, 0x0, 0x0, 0x1813c00)
	/usr/local/go/src/encoding/json/stream.go:134 +0x23d
encoding/json.(*Decoder).Decode(0xc421ae81e0, 0x18542c0, 0xc428f15180, 0x1c38800, 0xc428ebd500)
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc421ada1e0, 0xc427826000, 0x2000, 0x2600, 0xc428e975c0, 0x40, 0x38)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150 +0x295
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc4214c8190, 0x0, 0x1c37c80, 0xc428e975c0, 0x1a80, 0x1c38800, 0xc421aec0b8, 0x454dc0, 0xc421aec060)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x95
github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc421880e60, 0xc420ba1fa8, 0x7, 0x1c38800, 0xc428ebd500, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49 +0x7c
github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc421ada210)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:93 +0x12e
created by github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:60 +0xa8

goroutine 598 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc4213d20f0, 0x1c50c20, 0xc4212bb3c0, 0xc4200a4ae0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:114 +0x6a9
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func2(0xc420438000, 0xc4213d20f0, 0x1c50c20, 0xc4212bb3c0, 0xc4200a4ae0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:304 +0x73
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:302 +0x222

goroutine 599 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1(0xc422f926c0)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:75 +0x2f4
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func4(0x1ab44a0, 0xc4275ec600, 0x1ab44a0, 0xc427838000)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:109 +0x274
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc4218817c0, 0xc421881800, 0xc4218817e0, 0x1ab44a0, 0xc4275ec600, 0x1ab44a0, 0xc427838000)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc421cf0000, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 600 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc421cf0000, 0xc42128c540)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 609 [select, 2 minutes]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func1(0xc420494060, 0xc42128c000, 0xc42127b6e0, 0xc4202be1b0, 0xc42018e420, 0xc420494070)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:268 +0xf3
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:266 +0x7fe

goroutine 625 [chan receive, 2 minutes]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run(0xc4212400f0, 0x1c50c20, 0xc420047180, 0xc4200a4900)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:114 +0x6a9
github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func2(0xc420558010, 0xc4212400f0, 0x1c50c20, 0xc420047180, 0xc4200a4900)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:304 +0x73
created by github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:302 +0x222

goroutine 456 [select]:
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func1(0xc42443d170)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:75 +0x2f4
github.com/prometheus/prometheus/discovery/kubernetes.(*Pod).Run.func4(0x1ab44a0, 0xc4275d7800, 0x1ab44a0, 0xc427752300)
	/go/src/github.com/prometheus/prometheus/discovery/kubernetes/pod.go:109 +0x274
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnUpdate(0xc421de8c60, 0xc421de8ca0, 0xc421de8c80, 0x1ab44a0, 0xc4275d7800, 0x1ab44a0, 0xc427752300)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:199 +0x5d
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).run(0xc421c2c0b0, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:545 +0x24b
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:327 +0x30b

goroutine 457 [select]:
github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*processorListener).pop(0xc421c2c0b0, 0xc420458480)
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:519 +0x172
created by github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache.(*sharedIndexInformer).AddEventHandlerWithResyncPeriod
	/go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:328 +0x341

goroutine 658 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4249c8a80, 0x6, 0x6, 0x1620a780474, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421b3d380, 0xc4249c8a80, 0x6, 0x6, 0x146f, 0x1620a780474, 0x3ff0000000000000, 0xc4275ce360, 0xc4246e70a0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426378f20, 0xc4249c8a80, 0x6, 0x6, 0x146f, 0x1620a780474, 0x3ff0000000000000, 0xc420084060, 0x337b)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0080, 0xc427e00000, 0x16b37b, 0x30f489, 0xbea0b9031818a9de, 0x931e73814, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0080, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 659 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc426dc49c0, 0x6, 0x6, 0x1620a77b850, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc426dc0580, 0xc426dc49c0, 0x6, 0x6, 0x1620a77b850, 0x3ff0000000000000, 0xc426dc49c0, 0x6, 0x6)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc426ddd9c0, 0xc426dc49c0, 0x6, 0x6, 0x1620a77b850, 0x3ff0000000000000, 0x1c36b01, 0xc420084060, 0x2a98)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0180, 0xc42728e000, 0x154098, 0x2a2e00, 0xbea0b8fe3662cd98, 0x4a81993f9, 0x2883180, 0x56656372756f7365, 0x3a226e6f69737265, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0180, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 660 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425f0b500, 0x5, 0x6, 0x1620a77cc12, 0x401c000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4256d8000, 0xc425f0b500, 0x5, 0x6, 0x1620a77cc12, 0x401c000000000000, 0xc425f0b500, 0x5, 0x6)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42586bf20, 0xc425f0b500, 0x5, 0x6, 0x1620a77cc12, 0x401c000000000000, 0x1c36b01, 0xc420084060, 0x628d)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0680, 0xc426200000, 0x4e28d, 0x54400, 0xbea0b8ff79d5df05, 0x5d5929729, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0680, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 757 [select]:
net/http.(*persistConn).writeLoop(0xc42587c480)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 742 [select]:
net/http.(*persistConn).writeLoop(0xc421fa3560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 661 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4244b1b00, 0x4, 0x4, 0x1620a780dbc, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421e99400, 0xc4244b1b00, 0x4, 0x4, 0x21e8, 0x1620a780dbc, 0x0, 0xc425ca07e0, 0xc425825790)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42664c360, 0xc4244b1b00, 0x4, 0x4, 0x21e8, 0x1620a780dbc, 0x0, 0xc420084060, 0x5e3e)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0900, 0xc426a50000, 0x5e3e, 0x9ab9, 0xbea0b903ae87e891, 0x9bf8c0a9b, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0900, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 662 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc422d98e00, 0x7, 0x8, 0x1620a77c7de, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425dbc240, 0xc422d98e00, 0x7, 0x8, 0x1620a77c7de, 0x3ff0000000000000, 0xc422d98e00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42604f900, 0xc422d98e00, 0x7, 0x8, 0x1620a77c7de, 0x3ff0000000000000, 0x1c36b01, 0xc420084060, 0xe62)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0a00, 0xc427db7500, 0x217a, 0x2830, 0xbea0b8ff35578865, 0x5957976d1, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0a00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 663 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4244ea300, 0x7, 0x8, 0x1620a780943, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423e12ec0, 0xc4244ea300, 0x7, 0x8, 0x1e73, 0x1620a780943, 0x0, 0xc4262bad80, 0xc424f63820)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426674b20, 0xc4244ea300, 0x7, 0x8, 0x1e73, 0x1620a780943, 0x0, 0xc420084060, 0x93a)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0b00, 0xc426a32000, 0x893a, 0x9ab9, 0xbea0b90365e78ad6, 0x97b50e308, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0b00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 664 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fe2b40, 0x5, 0x5, 0x1620a77e0bf, 0x3f1d803af038592f, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4252ad5c0, 0xc425fe2b40, 0x5, 0x5, 0x1620a77e0bf, 0x3f1d803af038592f, 0xc425fe2b40, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42199b4e0, 0xc425fe2b40, 0x5, 0x5, 0x1620a77e0bf, 0x3f1d803af038592f, 0x1c36b01, 0xc420084060, 0x3a47)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0c00, 0xc4278bc000, 0x23a47, 0x2a100, 0xbea0b900cfb408fe, 0x711117d3b, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0c00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 665 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fe3040, 0x5, 0x5, 0x1620a77e931, 0x3f1f86a56d84b528, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc423e12080, 0xc425fe3040, 0x5, 0x5, 0x1620a77e931, 0x3f1f86a56d84b528, 0xc425fe3040, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4212a94c0, 0xc425fe3040, 0x5, 0x5, 0x1620a77e931, 0x3f1f86a56d84b528, 0x1c36b01, 0xc420084060, 0x59fa)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0d00, 0xc427004000, 0x259fa, 0x2a100, 0xbea0b90159627a63, 0x791f5827e, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0d00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 666 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4221acc80, 0x4, 0x4, 0x1620a77f6b0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423ca5940, 0xc4221acc80, 0x4, 0x4, 0x1, 0x1620a77f6b0, 0x0, 0xc424ddbcb0, 0xc4206afce0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4204273a0, 0xc4221acc80, 0x4, 0x4, 0x1, 0x1620a77f6b0, 0x0, 0xc420084060, 0x6801)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0e00, 0xc427030000, 0x6801, 0x9ab9, 0xbea0b902347962e3, 0x85fdcc961, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0e00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 667 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425851180, 0x5, 0x5, 0x1620a78116a, 0x41638e57c0000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421c67b40, 0xc425851180, 0x5, 0x5, 0x22f2, 0x1620a78116a, 0x41638e57c0000000, 0xc4262bb290, 0xc4259185b0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426d0de20, 0xc425851180, 0x5, 0x5, 0x22f2, 0x1620a78116a, 0x41638e57c0000000, 0xc420084060, 0x4fd5)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc0f00, 0xc427086000, 0x4fd5, 0x9ab9, 0xbea0b903eb0cd1df, 0x9f7abbde8, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc0f00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 668 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4247f4100, 0x7, 0x8, 0x1620a77ff5d, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421e00340, 0xc4247f4100, 0x7, 0x8, 0x12ba, 0x1620a77ff5d, 0x0, 0xc423e5a3f0, 0xc4249ba3c0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42502ba20, 0xc4247f4100, 0x7, 0x8, 0x12ba, 0x1620a77ff5d, 0x0, 0xc420084060, 0x7c13)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1000, 0xc4269f2000, 0x7c13, 0x9ab9, 0xbea0b902c60afa02, 0x8e43ebe6a, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1000, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 669 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425c09c00, 0x7, 0x8, 0x1620a7814e5, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421abad00, 0xc425c09c00, 0x7, 0x8, 0x2571, 0x1620a7814e5, 0x0, 0xc423e5a6c0, 0xc425ab7200)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426d29c20, 0xc425c09c00, 0x7, 0x8, 0x2571, 0x1620a7814e5, 0x0, 0xc420084060, 0x490)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1100, 0xc4270be000, 0x8490, 0x9ab9, 0xbea0b9042497ca10, 0xa2cd1804a, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1100, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 670 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425ccbe00, 0x7, 0x8, 0x1620a77eae7, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4254112c0, 0xc425ccbe00, 0x7, 0x8, 0x1620a77eae7, 0x0, 0xc425ccbe00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4219130c0, 0xc425ccbe00, 0x7, 0x8, 0x1620a77eae7, 0x0, 0x1c36b01, 0xc420084060, 0x1db8)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1200, 0xc4278f8000, 0x9db8, 0xa6c0, 0xbea0b90173721587, 0x7ac051d9b, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1200, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 671 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fe2320, 0x5, 0x5, 0x1620a77c4c6, 0x7ff8000000000001, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425f6d380, 0xc425fe2320, 0x5, 0x5, 0x1620a77c4c6, 0x7ff8000000000001, 0xc425fe2320, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc426181da0, 0xc425fe2320, 0x5, 0x5, 0x1620a77c4c6, 0x7ff8000000000001, 0x1c36b01, 0xc420084060, 0x1eb6)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1300, 0xc426162000, 0x7116, 0xa6c0, 0xbea0b8ff0616d27d, 0x56638c08d, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1300, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 672 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc42443e400, 0x7, 0x8, 0x1620a780885, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423e12dc0, 0xc42443e400, 0x7, 0x8, 0x1e2d, 0x1620a780885, 0x3ff0000000000000, 0xc4262bac60, 0xc424493b60)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4266749c0, 0xc42443e400, 0x7, 0x8, 0x1e2d, 0x1620a780885, 0x3ff0000000000000, 0xc420084060, 0x2167)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1400, 0xc426023500, 0x2167, 0x3393, 0xbea0b9035a92d107, 0x96ffc295b, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1400, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 689 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fb5d00, 0x7, 0x8, 0x1620a77ce77, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425812380, 0xc425fb5d00, 0x7, 0x8, 0x1620a77ce77, 0x0, 0xc425fb5d00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc425864c80, 0xc425fb5d00, 0x7, 0x8, 0x1620a77ce77, 0x0, 0x1c36b01, 0xc420084060, 0x12e7)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1500, 0xc424a8e000, 0x92e7, 0xa6c0, 0xbea0b8ffa2c71613, 0x5fa1e982e, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1500, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 690 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc426de0280, 0x5, 0x5, 0x1620a77b886, 0x3f141807552b4d02, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc426dc03c0, 0xc426de0280, 0x5, 0x5, 0x1620a77b886, 0x3f141807552b4d02, 0xc426de0280, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc426ddd7e0, 0xc426de0280, 0x5, 0x5, 0x1620a77b886, 0x3f141807552b4d02, 0x1c36b01, 0xc420084060, 0x2416)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1600, 0xc426fd2000, 0x4c46, 0x5260, 0xbea0b8fe399a2fcd, 0x4ab50f63f, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1600, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 691 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4251ee5a0, 0x5, 0x5, 0x1620a77ce2d, 0x3f1b0934a0e04ea4, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4255206c0, 0xc4251ee5a0, 0x5, 0x5, 0x1620a77ce2d, 0x3f1b0934a0e04ea4, 0xc4251ee5a0, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4257a63c0, 0xc4251ee5a0, 0x5, 0x5, 0x1620a77ce2d, 0x3f1b0934a0e04ea4, 0x1c36b01, 0xc420084060, 0x347d)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1700, 0xc423046000, 0x2347d, 0x2a100, 0xbea0b8ff9e639771, 0x5f5bb19b3, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1700, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 692 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc42481bea0, 0x5, 0x5, 0x1620a77f1a1, 0x3f173d55c34d698d, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc423ca4440, 0xc42481bea0, 0x5, 0x5, 0x1620a77f1a1, 0x3f173d55c34d698d, 0xc42481bea0, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42186aa20, 0xc42481bea0, 0x5, 0x5, 0x1620a77f1a1, 0x3f173d55c34d698d, 0x1c36b01, 0xc420084060, 0xf38)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1800, 0xc427950000, 0x3768, 0x5260, 0xbea0b901e2e73c1d, 0x812afd89d, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1800, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 693 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4251ee780, 0x5, 0x5, 0x1620a77d567, 0x3f12725108e4b3a7, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425520880, 0xc4251ee780, 0x5, 0x5, 0x1620a77d567, 0x3f12725108e4b3a7, 0xc4251ee780, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4257a6840, 0xc4251ee780, 0x5, 0x5, 0x1620a77d567, 0x3f12725108e4b3a7, 0x1c36b01, 0xc420084060, 0x1e52)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1900, 0xc422e1aa00, 0x4682, 0x5260, 0xbea0b9001568e9a0, 0x663f5ffb6, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1900, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 694 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4200bd720, 0x5, 0x5, 0x1620a77f8e4, 0x4192e73938000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423ca5a40, 0xc4200bd720, 0x5, 0x5, 0x275, 0x1620a77f8e4, 0x4192e73938000000, 0xc424ddbdd0, 0xc4218ae8f0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc420427640, 0xc4200bd720, 0x5, 0x5, 0x275, 0x1620a77f8e4, 0x4192e73938000000, 0xc420084060, 0x4f53)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1a00, 0xc427044000, 0x4f53, 0x9ab9, 0xbea0b9025a7b2261, 0x8817952d7, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1a00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 695 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425df0280, 0x5, 0x5, 0x1620a7816fc, 0x7ff8000000000001, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421abbb80, 0xc425df0280, 0x5, 0x5, 0x26cb, 0x1620a7816fc, 0x7ff8000000000001, 0xc423e5a900, 0xc425aba110)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4277f9b80, 0xc425df0280, 0x5, 0x5, 0x26cb, 0x1620a7816fc, 0x7ff8000000000001, 0xc420084060, 0x7109)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1b00, 0xc42710c000, 0x7109, 0x9ab9, 0xbea0b90448d2b8f4, 0xa4ca7391d, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1b00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 696 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425e5fd00, 0x7, 0x8, 0x1620a77e47a, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc422f00ec0, 0xc425e5fd00, 0x7, 0x8, 0x1620a77e47a, 0x3ff0000000000000, 0xc425e5fd00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc421a23a00, 0xc425e5fd00, 0x7, 0x8, 0x1620a77e47a, 0x3ff0000000000000, 0x1c36b01, 0xc420084060, 0xe73)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1c00, 0xc425754000, 0x218b, 0x2830, 0xbea0b9010d00075a, 0x749f84582, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1c00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 697 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fe2500, 0x5, 0x5, 0x1620a77c650, 0x415b2b0580000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425f6d5c0, 0xc425fe2500, 0x5, 0x5, 0x1620a77c650, 0x415b2b0580000000, 0xc425fe2500, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc425feab00, 0xc425fe2500, 0x5, 0x5, 0x1620a77c650, 0x415b2b0580000000, 0x1c36b01, 0xc420084060, 0x1927)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1d00, 0xc423ce3500, 0x4157, 0x5260, 0xbea0b8ff1d954099, 0x57db72ebe, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1d00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 698 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4253c9600, 0x7, 0x8, 0x1620a780af4, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423e13140, 0xc4253c9600, 0x7, 0x8, 0x20a7, 0x1620a780af4, 0x0, 0xc4262bb050, 0xc424f67500)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426675680, 0xc4253c9600, 0x7, 0x8, 0x20a7, 0x1620a780af4, 0x0, 0xc420084060, 0x12e3)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1e00, 0xc426a46000, 0x92e3, 0x9ab9, 0xbea0b903841bf63c, 0x99520186e, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1e00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 699 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4245e7d00, 0x7, 0x8, 0x1620a77ff55, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421adfcc0, 0xc4245e7d00, 0x7, 0x8, 0x1215, 0x1620a77ff55, 0x0, 0xc427572360, 0xc424675f00)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42635c220, 0xc4245e7d00, 0x7, 0x8, 0x1215, 0x1620a77ff55, 0x0, 0xc420084060, 0x7dd)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc420fc1f00, 0xc4269e8000, 0x87dd, 0x9ab9, 0xbea0b902c591b6b8, 0x8e3c57b18, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc420fc1f00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 700 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc422050780, 0x5, 0x5, 0x1620a77fcad, 0x7ff0000000000002, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc421919b00, 0xc422050780, 0x5, 0x5, 0x1620a77fcad, 0x7ff0000000000002, 0x0, 0xc42003b800, 0x1)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42719b1a0, 0xc422050780, 0x5, 0x5, 0x1620a77fcad, 0x7ff0000000000002, 0xc4217c3ba0, 0xc42217d990, 0xc42217d918)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append.func1(0xc422050780, 0x5, 0x5, 0x28a5500)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:937 +0x77
github.com/prometheus/prometheus/scrape.(*scrapeCache).forEachStale(0xc422044930, 0xc42217dcd8)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:595 +0x112
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e000, 0xc4269c0000, 0x0, 0x9ab9, 0xbea0b90298a0df15, 0x8bb39d94d, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:935 +0xc41
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e000, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 701 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc422ebc500, 0x5, 0x5, 0x1620a77ff15, 0x7ff8000000000001, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421e00200, 0xc422ebc500, 0x5, 0x5, 0x10ff, 0x1620a77ff15, 0x7ff8000000000001, 0xc423e5a2d0, 0xc424672220)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42502b860, 0xc422ebc500, 0x5, 0x5, 0x10ff, 0x1620a77ff15, 0x7ff8000000000001, 0xc420084060, 0x62c3)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e100, 0xc4269de000, 0x62c3, 0x9ab9, 0xbea0b902c1c01576, 0x8dff3d999, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e100, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 702 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425ca5600, 0x7, 0x8, 0x1620a77f38e, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc424461000, 0xc425ca5600, 0x7, 0x8, 0x1620a77f38e, 0x3ff0000000000000, 0xc425ca5600, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4218f3660, 0xc425ca5600, 0x7, 0x8, 0x1620a77f38e, 0x3ff0000000000000, 0x1c36b01, 0xc420084060, 0xe63)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e200, 0xc4276ad500, 0x217b, 0x2830, 0xbea0b90204ae4021, 0x83011a65a, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e200, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 703 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425ffe280, 0x5, 0x5, 0x1620a77c352, 0x4191d5d9c0000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc426171fc0, 0xc425ffe280, 0x5, 0x5, 0x1620a77c352, 0x4191d5d9c0000000, 0xc425ffe280, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42604f660, 0xc425ffe280, 0x5, 0x5, 0x1620a77c352, 0x4191d5d9c0000000, 0x1c36b01, 0xc420084060, 0x2744)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e300, 0xc420134a00, 0x4f74, 0x5260, 0xbea0b8feeb86f78d, 0x5500e1b9c, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e300, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 704 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc42445a140, 0x5, 0x5, 0x1620a781ca2, 0x4196154708000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc4219ddb00, 0xc42445a140, 0x5, 0x5, 0x29d2, 0x1620a781ca2, 0x4196154708000000, 0xc423e5abd0, 0xc4235ecf70)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42712b400, 0xc42445a140, 0x5, 0x5, 0x29d2, 0x1620a781ca2, 0x4196154708000000, 0xc420084060, 0x4f56)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e400, 0xc4271e4000, 0x4f56, 0x9ab9, 0xbea0b904a374eaf1, 0xaa2e43515, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e400, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 705 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4223ab400, 0x5, 0x5, 0x1620a77f982, 0x3f1409aad7373c8b, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423ca5b40, 0xc4223ab400, 0x5, 0x5, 0x320, 0x1620a77f982, 0x3f1409aad7373c8b, 0xc424ddbef0, 0xc42198f790)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4204278e0, 0xc4223ab400, 0x5, 0x5, 0x320, 0x1620a77f982, 0x3f1409aad7373c8b, 0xc420084060, 0x4c55)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e500, 0xc42704e000, 0x4c55, 0x9ab9, 0xbea0b90263e7d915, 0x88ae60946, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e500, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 706 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc423e09500, 0x7, 0x8, 0x1620a77fdc6, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421ade9c0, 0xc423e09500, 0x7, 0x8, 0xb03, 0x1620a77fdc6, 0x0, 0xc427572240, 0xc4206b8060)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42519f340, 0xc423e09500, 0x7, 0x8, 0xb03, 0x1620a77fdc6, 0x0, 0xc420084060, 0x7735)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e600, 0xc4269ca000, 0x7735, 0x9ab9, 0xbea0b902a96856d7, 0x8cc015120, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e600, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 707 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc423922400, 0x7, 0x8, 0x1620a780664, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421adfe80, 0xc423922400, 0x7, 0x8, 0x1ad7, 0x1620a780664, 0x0, 0xc427572510, 0xc42522ebd0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42635cc40, 0xc423922400, 0x7, 0x8, 0x1ad7, 0x1620a780664, 0x0, 0xc420084060, 0x4ced)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e700, 0xc4269fc000, 0xcced, 0x1d02b, 0xbea0b90335ae42e0, 0x94f7cd146, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e700, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 708 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425c1ee60, 0x5, 0x5, 0x1620a782a34, 0x3f1286fc6f380261, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421480640, 0xc425c1ee60, 0x5, 0x5, 0x41e9, 0x1620a782a34, 0x3f1286fc6f380261, 0xc423e5aea0, 0xc426c6c680)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42723f2e0, 0xc425c1ee60, 0x5, 0x5, 0x41e9, 0x1620a782a34, 0x3f1286fc6f380261, 0xc420084060, 0x4679)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e800, 0xc427254000, 0x4679, 0x9ab9, 0xbea0b9058416bda3, 0xb71f12fcf, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e800, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 709 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc426190800, 0x7, 0x8, 0x1620a77c443, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425f6d1c0, 0xc426190800, 0x7, 0x8, 0x1620a77c443, 0x0, 0xc426190800, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc426181b60, 0xc426190800, 0x7, 0x8, 0x1620a77c443, 0x0, 0x1c36b01, 0xc420084060, 0x12e2)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204e900, 0xc426184000, 0x92e2, 0xa6c0, 0xbea0b8fef9e1ef6e, 0x55e6913c2, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204e900, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 710 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4251ee280, 0x5, 0x5, 0x1620a77d073, 0x4162e3a700000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425520440, 0xc4251ee280, 0x5, 0x5, 0x1620a77d073, 0x4162e3a700000000, 0xc4251ee280, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4257d1460, 0xc4251ee280, 0x5, 0x5, 0x1620a77d073, 0x4162e3a700000000, 0x1c36b01, 0xc420084060, 0x1934)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ea00, 0xc422e10000, 0x4164, 0x5260, 0xbea0b8ffc572ab5a, 0x61864f795, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ea00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 711 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4253fcbe0, 0x5, 0x5, 0x1620a781cc1, 0x3f103fcf5f098c01, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc4219ddd00, 0xc4253fcbe0, 0x5, 0x5, 0x2a7d, 0x1620a781cc1, 0x3f103fcf5f098c01, 0xc423e5acf0, 0xc423705450)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42712b4e0, 0xc4253fcbe0, 0x5, 0x5, 0x2a7d, 0x1620a781cc1, 0x3f103fcf5f098c01, 0xc420084060, 0x4677)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204eb00, 0xc4271f2000, 0x4677, 0x9ab9, 0xbea0b904a546da93, 0xaa4b624e7, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204eb00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 712 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4261c4380, 0x4, 0x4, 0x1620a77c347, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425f1a140, 0xc4261c4380, 0x4, 0x4, 0x1620a77c347, 0x0, 0xc4261c4380, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc426005020, 0xc4261c4380, 0x4, 0x4, 0x1620a77c347, 0x0, 0x1c36b01, 0xc420084060, 0x586)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ec00, 0xc426196000, 0x57e6, 0xa6c0, 0xbea0b8feeaddc527, 0x54f64e949, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ec00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 713 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4247f4400, 0x7, 0x8, 0x1620a781b23, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc4219dd8c0, 0xc4247f4400, 0x7, 0x8, 0x298c, 0x1620a781b23, 0x3ff0000000000000, 0xc423e5aab0, 0xc425fe60d0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42712b340, 0xc4247f4400, 0x7, 0x8, 0x298c, 0x1620a781b23, 0x3ff0000000000000, 0xc420084060, 0x2171)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ed00, 0xc4237cc000, 0x2171, 0x3393, 0xbea0b9048c96bb99, 0xa8c060601, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ed00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 714 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc424807400, 0x7, 0x8, 0x1620a7820dd, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421919900, 0xc424807400, 0x7, 0x8, 0x2b25, 0x1620a7820dd, 0x0, 0xc427572a20, 0xc425226ed0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42719abe0, 0xc424807400, 0x7, 0x8, 0x2b25, 0x1620a7820dd, 0x0, 0xc420084060, 0x2d70)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ee00, 0xc427200000, 0xad70, 0x1d02b, 0xbea0b904e862a9e5, 0xae36cbe26, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ee00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 715 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4254d3b00, 0x7, 0x8, 0x1620a7819e8, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc4219962c0, 0xc4254d3b00, 0x7, 0x8, 0x280b, 0x1620a7819e8, 0x0, 0xc427572900, 0xc4249bf270)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc427141980, 0xc4254d3b00, 0x7, 0x8, 0x280b, 0x1620a7819e8, 0x0, 0xc420084060, 0x2606)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ef00, 0xc427148000, 0xa606, 0x1d02b, 0xbea0b904756d64ac, 0xa7941e4bd, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ef00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 716 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4259b5400, 0x7, 0x8, 0x1620a77ed2e, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc424ac2100, 0xc4259b5400, 0x7, 0x8, 0x1620a77ed2e, 0x0, 0xc4259b5400, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4218eeb40, 0xc4259b5400, 0x7, 0x8, 0x1620a77ed2e, 0x0, 0x1c36b01, 0xc420084060, 0x1ee1)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f000, 0xc427918000, 0xc5a1, 0x14f80, 0xbea0b9019a9b1f18, 0x7cec8f139, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f000, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 717 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc42473c3c0, 0x5, 0x5, 0x1620a782827, 0x3f141659d601a137, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc4214805c0, 0xc42473c3c0, 0x5, 0x5, 0x3270, 0x1620a782827, 0x3f141659d601a137, 0xc423e5ae10, 0xc423c8c820)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42723eca0, 0xc42473c3c0, 0x5, 0x5, 0x3270, 0x1620a782827, 0x3f141659d601a137, 0xc420084060, 0x4c57)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f100, 0xc427246000, 0x4c57, 0x9ab9, 0xbea0b905606982ce, 0xb52a92b07, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f100, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 718 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425d94a00, 0x7, 0x8, 0x1620a77da9f, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425813640, 0xc425d94a00, 0x7, 0x8, 0x1620a77da9f, 0x0, 0xc425d94a00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4255cadc0, 0xc425d94a00, 0x7, 0x8, 0x1620a77da9f, 0x0, 0x1c36b01, 0xc420084060, 0x2451)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f200, 0xc421e40000, 0xa451, 0xa6c0, 0xbea0b900696e0cf6, 0x6b395ed38, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f200, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 719 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc424ed2320, 0x5, 0x5, 0x1620a77cafe, 0x3f1d62192d562d11, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc42564adc0, 0xc424ed2320, 0x5, 0x5, 0x1620a77cafe, 0x3f1d62192d562d11, 0xc424ed2320, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42580ca80, 0xc424ed2320, 0x5, 0x5, 0x1620a77cafe, 0x3f1d62192d562d11, 0x1c36b01, 0xc420084060, 0x5da5)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f300, 0xc427dbe000, 0x25da5, 0x2a100, 0xbea0b8ff695f935d, 0x5c51c4b91, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f300, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 720 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4204f0f00, 0x7, 0x8, 0x1620a77f766, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423e12280, 0xc4204f0f00, 0x7, 0x8, 0x142, 0x1620a77f766, 0x0, 0xc4262ba990, 0xc4206b6190)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4212a9bc0, 0xc4204f0f00, 0x7, 0x8, 0x142, 0x1620a77f766, 0x0, 0xc420084060, 0x94c)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f400, 0xc42703a000, 0x894c, 0x9ab9, 0xbea0b90243ba6373, 0x86ab89387, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f400, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 721 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc424293c80, 0x4, 0x4, 0x1620a780a45, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc423e13040, 0xc424293c80, 0x4, 0x4, 0x1fa6, 0x1620a780a45, 0x3ff0000000000000, 0xc4262baf30, 0xc4249bd2d0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426675500, 0xc424293c80, 0x4, 0x4, 0x1fa6, 0x1620a780a45, 0x3ff0000000000000, 0xc420084060, 0x5b91)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f500, 0xc426a3c000, 0x5b91, 0x9ab9, 0xbea0b903754309c4, 0x98aac61f6, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f500, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 722 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc424ad0100, 0x7, 0x8, 0x1620a781af7, 0x3ff0000000000000, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421a73b40, 0xc424ad0100, 0x7, 0x8, 0x297e, 0x1620a781af7, 0x3ff0000000000000, 0xc427100120, 0xc4222cf270)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc4271baa20, 0xc424ad0100, 0x7, 0x8, 0x297e, 0x1620a781af7, 0x3ff0000000000000, 0xc420084060, 0x219a)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f600, 0xc425a26000, 0x219a, 0x3393, 0xbea0b9048a02c834, 0xa8972124c, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f600, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 723 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425730500, 0x7, 0x8, 0x1620a781477, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421c67d00, 0xc425730500, 0x7, 0x8, 0x239c, 0x1620a781477, 0x0, 0xc4262bb3b0, 0xc425bdabe0)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc426d0df40, 0xc425730500, 0x7, 0x8, 0x239c, 0x1620a781477, 0x0, 0xc420084060, 0x4c39)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f700, 0xc427096000, 0xcc39, 0x1d02b, 0xbea0b9041e05b6af, 0xa263f6d00, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f700, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 724 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425bf9600, 0x4, 0x4, 0x1620a77e936, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc421e994c0, 0xc425bf9600, 0x4, 0x4, 0x1620a77e936, 0x0, 0xc425bf9600, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42664c4c0, 0xc425bf9600, 0x4, 0x4, 0x1620a77e936, 0x0, 0x2883180, 0xed23465e7, 0xbea0b999d9b3d792)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).addReportSample(0xc42204f800, 0x1c50f20, 0xc42664c4c0, 0x1adb825, 0x3, 0x1620a77e936, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:1055 +0x1d8
github.com/prometheus/prometheus/scrape.(*scrapeLoop).report(0xc42204f800, 0xbea0b90159aff1a2, 0x79242f9b0, 0x2883180, 0x2540fc20a, 0x0, 0x0, 0x1c3a3c0, 0x28a3fc0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:984 +0x1e8
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f800, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:711 +0x809
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 725 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc422d99a00, 0x7, 0x8, 0x1620a77c8eb, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425dbc380, 0xc422d99a00, 0x7, 0x8, 0x1620a77c8eb, 0x0, 0xc422d99a00, 0x7, 0x8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42604fc80, 0xc422d99a00, 0x7, 0x8, 0x1620a77c8eb, 0x0, 0x1c36b01, 0xc420084060, 0x1cd8)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204f900, 0xc425f72000, 0x9cd8, 0xa6c0, 0xbea0b8ff49be3e00, 0x5a57af634, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204f900, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 726 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc426c67800, 0x7, 0x8, 0x1620a782ba0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421480880, 0xc426c67800, 0x7, 0x8, 0x4291, 0x1620a782ba0, 0x0, 0xc423e5afc0, 0xc42582d150)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc42723f3c0, 0xc426c67800, 0x7, 0x8, 0x4291, 0x1620a782ba0, 0x0, 0xc420084060, 0x932)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204fa00, 0xc427264000, 0x8932, 0x9ab9, 0xbea0b90599c938a1, 0xb87a3aaf6, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204fa00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 727 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc4214e3380, 0x3, 0x3, 0x1620a782797, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast(0xc421a16980, 0xc4214e3380, 0x3, 0x3, 0x2d01, 0x1620a782797, 0x0, 0xc4275ce6c0, 0xc423efb850)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:153 +0x129
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast(0xc427834a80, 0xc4214e3380, 0x3, 0x3, 0x2d01, 0x1620a782797, 0x0, 0xc420084060, 0x1737)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:254 +0xac
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204fc00, 0xc428110000, 0xa1737, 0x105183, 0xbea0b90557d551ad, 0xb4a14f9e3, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:831 +0x1a6d
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204fc00, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 728 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425ebb880, 0x4, 0x4, 0x1620a77da93, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc425813740, 0xc425ebb880, 0x4, 0x4, 0x1620a77da93, 0x0, 0xc425ebb880, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc4255cb060, 0xc425ebb880, 0x4, 0x4, 0x1620a77da93, 0x0, 0x1c36b01, 0xc420084060, 0x102d)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204fe80, 0xc4240a2000, 0x15fad, 0x2a100, 0xbea0b90068be386a, 0x6b2e618b6, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204fe80, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 729 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425f12e80, 0x4, 0x4, 0x1620a77ceaf, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc42564a9c0, 0xc425f12e80, 0x4, 0x4, 0x1620a77ceaf, 0x0, 0xc425f12e80, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42580c760, 0xc425f12e80, 0x4, 0x4, 0x1620a77ceaf, 0x0, 0x1c36b01, 0xc420084060, 0xe6e)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc42204ff80, 0xc426544000, 0x15dee, 0x2a100, 0xbea0b8ffa61c4754, 0x5fd73c978, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc42204ff80, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 730 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425d64f80, 0x4, 0x4, 0x1620a77e5df, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4252ad3c0, 0xc425d64f80, 0x4, 0x4, 0x1620a77e5df, 0x0, 0xc425d64f80, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc42199b160, 0xc425d64f80, 0x4, 0x4, 0x1620a77e5df, 0x0, 0x1c36b01, 0xc420084060, 0xbaf)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc4222cc080, 0xc42787a000, 0x15b2f, 0x2a100, 0xbea0b9012252d480, 0x75f4b12a0, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc4222cc080, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 731 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425f13b00, 0x4, 0x4, 0x1620a77e5d7, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4244609c0, 0xc425f13b00, 0x4, 0x4, 0x1620a77e5d7, 0x0, 0xc425f13b00, 0x4, 0x4)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc421901880, 0xc425f13b00, 0x4, 0x4, 0x1620a77e5d7, 0x0, 0x1c36b01, 0xc420084060, 0xa12)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc4222cc180, 0xc42784e000, 0x15992, 0x2a100, 0xbea0b90121cfc07f, 0x75ec7feb3, 0x2883180, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc4222cc180, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 459 [select]:
net/http.(*persistConn).writeLoop(0xc421a41200)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 739 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a47b0, 0x72, 0xc421891960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42162f698, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42162f698, 0xc4223ed000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42162f680, 0xc4223ed000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42162f680, 0xc4223ed000, 0x1000, 0x1000, 0xc422f503d8, 0xc4218f4480, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42017e180, 0xc4223ed000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4218f4480, 0xc4223ed000, 0x1000, 0x1000, 0xc421891b98, 0x403855, 0xc421cee4e0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c360)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c360, 0x1, 0x0, 0x0, 0x0, 0xc4240953e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4218f4480)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 676 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4950, 0x72, 0xc421c37960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42033cb98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42033cb98, 0xc421e5f000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42033cb80, 0xc421e5f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42033cb80, 0xc421e5f000, 0x1000, 0x1000, 0xc422f50408, 0xc4223ca120, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc420178028, 0xc421e5f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4223ca120, 0xc421e5f000, 0x1000, 0x1000, 0xc421c37b98, 0x403855, 0xc42127a2a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421da9c80)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421da9c80, 0x1, 0x0, 0x0, 0x0, 0xc421cc7d40, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4223ca120)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 677 [select]:
net/http.(*persistConn).writeLoop(0xc4223ca120)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 741 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4af0, 0x72, 0xc422c70960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc421b87c18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc421b87c18, 0xc421ff4000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc421b87c00, 0xc421ff4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc421b87c00, 0xc421ff4000, 0x1000, 0x1000, 0xc426de4140, 0xc421fa3560, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4213e31b8, 0xc421ff4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc421fa3560, 0xc421ff4000, 0x1000, 0x1000, 0xc422c70b98, 0x403855, 0xc421cee780)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9d740)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9d740, 0x1, 0x0, 0x0, 0x0, 0xc4230188a0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc421fa3560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 807 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d6040, 0x72, 0xc425dfa960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f63218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f63218, 0xc425f9c000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f63200, 0xc425f9c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f63200, 0xc425f9c000, 0x1000, 0x1000, 0xc422f504f8, 0xc425f7f440, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425e95958, 0xc425f9c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425f7f440, 0xc425f9c000, 0x1000, 0x1000, 0xc425dfab98, 0x403855, 0xc421b21a40)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421383500)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421383500, 0x1, 0x0, 0x0, 0x0, 0xc427170000, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425f7f440)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1162 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc424ad2ba0, 0x1620a7885d0, 0x1620a7885d0, 0x1c4f0a0, 0xc421897020, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc424ad2ba0, 0x1620a7885d0, 0x1620a7885d0, 0x10e7c3f408, 0x2883180, 0x1f025da4282c8f50, 0xc4282c8ef8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc424ad2ba0, 0x7f9a0946e1d8, 0xc42016a380, 0xc428c12140, 0xc425521958, 0xc425521960, 0x0, 0x492486)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc424ad2ba0, 0xc428c12190, 0xc428c12140, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc424ad2ba0, 0xc428c12190, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc428c12190, 0x1c50c20, 0xc4248c9ac0, 0xc428246d49)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/web/api/v1.(*API).query(0xc420573ab0, 0xc42174a300, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:236 +0x2b2
github.com/prometheus/prometheus/web/api/v1.(*API).(github.com/prometheus/prometheus/web/api/v1.query)-fm(0xc42174a300, 0xc4283b9700, 0xc428c120f0, 0x3)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:174 +0x34
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1c45ca0, 0xc4283b9700, 0xc42174a300)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:159 +0x55
net/http.HandlerFunc.ServeHTTP(0xc4202fe460, 0x1c45ca0, 0xc4283b9700, 0xc42174a300)
	/usr/local/go/src/net/http/server.go:1947 +0x44
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1c3b7e0, 0xc4202fe460, 0x1c474e0, 0xc428c120f0, 0xc42174a300)
	/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x7c
net/http.(Handler).ServeHTTP-fm(0x1c474e0, 0xc428c120f0, 0xc42174a300)
	/usr/local/go/src/net/http/h2_bundle.go:5475 +0x4d
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc4283b96e0, 0xc42174a300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc4283b96e0, 0xc42174a300)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc4283b96e0, 0xc42174a200, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc4206f0540, 0x1c47420, 0xc4283b96e0, 0xc42174a200)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc420126740, 0x1c47420, 0xc4283b96e0, 0xc42174a200)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.StripPrefix.func1(0x1c47420, 0xc4283b96e0, 0xc42174a100)
	/usr/local/go/src/net/http/server.go:1986 +0x19a
net/http.HandlerFunc.ServeHTTP(0xc420f1e2d0, 0x1c47420, 0xc4283b96e0, 0xc42174a100)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc4283b96e0, 0xc42174a100)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc420c036c0, 0xc42174a000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc420c036c0, 0xc42174a000)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc420c036c0, 0xc42174a000)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc4251ef0e0, 0x1c50c20, 0xc422cad380)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 682 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4c90, 0x72, 0xc42132c960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc423ceeb98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc423ceeb98, 0xc42206f000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc423ceeb80, 0xc42206f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc423ceeb80, 0xc42206f000, 0x1000, 0x1000, 0xc422f50450, 0xc423cc7200, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc420179c18, 0xc42206f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc423cc7200, 0xc42206f000, 0x1000, 0x1000, 0xc42132cb98, 0x403855, 0xc421effa40)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42018ff80)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42018ff80, 0x1, 0x0, 0x0, 0x0, 0xc423e7e300, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc423cc7200)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 683 [select]:
net/http.(*persistConn).writeLoop(0xc423cc7200)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 460 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4610, 0x72, 0xc422c6b960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc421205398, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc421205398, 0xc424586000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc421205380, 0xc424586000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc421205380, 0xc424586000, 0x1000, 0x1000, 0xc42773e180, 0xc42176a900, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424582000, 0xc424586000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42176a900, 0xc424586000, 0x1000, 0x1000, 0xc422c6bb98, 0x403855, 0xc4200a43c0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4204e4180)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4204e4180, 0x1, 0x0, 0x0, 0x0, 0xc423e7ede0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42176a900)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 686 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4540, 0x72, 0xc4215b6960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4246c4218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4246c4218, 0xc4246d0000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4246c4200, 0xc4246d0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4246c4200, 0xc4246d0000, 0x1000, 0x1000, 0xc422f50468, 0xc4246cc000, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc422ec2018, 0xc4246d0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4246cc000, 0xc4246d0000, 0x1000, 0x1000, 0xc4215b6b98, 0x403855, 0xc42127ab40)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421649980)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421649980, 0x1, 0x0, 0x0, 0x0, 0xc423e7ef00, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4246cc000)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 687 [select]:
net/http.(*persistConn).writeLoop(0xc4246cc000)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 754 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4470, 0x72, 0xc422c6c960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc421205698, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc421205698, 0xc4246d3000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc421205680, 0xc4246d3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc421205680, 0xc4246d3000, 0x1000, 0x1000, 0xc42773e1b0, 0xc42176ac60, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc422ec2050, 0xc4246d3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42176ac60, 0xc4246d3000, 0x1000, 0x1000, 0xc422c6cb98, 0x403855, 0xc4200a4f00)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4200394a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4200394a0, 0x1, 0x0, 0x0, 0x0, 0xc4230189c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42176ac60)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 755 [select]:
net/http.(*persistConn).writeLoop(0xc42176ac60)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 761 [select]:
net/http.(*persistConn).writeLoop(0xc4259557a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 841 [chan receive, 1 minutes]:
github.com/prometheus/prometheus/web.(*Handler).reload(0xc42033ea00, 0x1c47420, 0xc426e08020, 0xc422d71b00)
	/go/src/github.com/prometheus/prometheus/web/web.go:646 +0x85
github.com/prometheus/prometheus/web.(*Handler).(github.com/prometheus/prometheus/web.reload)-fm(0x1c47420, 0xc426e08020, 0xc422d71b00)
	/go/src/github.com/prometheus/prometheus/web/web.go:241 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc426e08020, 0xc422d71a00, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc42016aac0, 0x1c47420, 0xc426e08020, 0xc422d71a00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4201264a0, 0x1c47420, 0xc426e08020, 0xc422d71a00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc426e08020, 0xc422d71a00)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc424e501c0, 0xc422d71900)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc424e501c0, 0xc422d71900)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc424e501c0, 0xc422d71900)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc424bab680, 0x1c50c20, 0xc423ca5240)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 789 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d62b0, 0x72, 0xc423da5960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425647698, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425647698, 0xc425962000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425647680, 0xc425962000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425647680, 0xc425962000, 0x1000, 0x1000, 0xc4232553a8, 0xc4256a0a20, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425885018, 0xc425962000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4256a0a20, 0xc425962000, 0x1000, 0x1000, 0xc423da5b98, 0x403855, 0xc421b9afc0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4205c8e40)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4205c8e40, 0x1, 0x0, 0x0, 0x0, 0xc423e7f440, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4256a0a20)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 769 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a43a0, 0x72, 0xc4209c8960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc424aa0798, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc424aa0798, 0xc424a9b000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc424aa0780, 0xc424a9b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc424aa0780, 0xc424a9b000, 0x1000, 0x1000, 0xc426de41e0, 0xc424aafe60, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424a60a28, 0xc424a9b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc424aafe60, 0xc424a9b000, 0x1000, 0x1000, 0xc4209c8b98, 0x403855, 0xc421f374a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc424ad3500)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc424ad3500, 0x1, 0x0, 0x0, 0x0, 0xc42767e960, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc424aafe60)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 770 [select]:
net/http.(*persistConn).writeLoop(0xc424aafe60)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 648 [select]:
net/http.(*persistConn).writeLoop(0xc4244df0e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 645 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4200, 0x72, 0xc423da1960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42443a218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42443a218, 0xc424458000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42443a200, 0xc424458000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42443a200, 0xc424458000, 0x1000, 0x1000, 0xc423255340, 0xc424454000, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4247a04f8, 0xc424458000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc424454000, 0xc424458000, 0x1000, 0x1000, 0xc423da1b98, 0x403855, 0xc42127aa80)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c420)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c420, 0x1, 0x0, 0x0, 0x0, 0xc424014cc0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc424454000)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 810 [select]:
net/http.(*persistConn).writeLoop(0xc424640120)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 647 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a094a4130, 0x72, 0xc424451960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4244b1a18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4244b1a18, 0xc4244f6000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4244b1a00, 0xc4244f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4244b1a00, 0xc4244f6000, 0x1000, 0x1000, 0xc423255348, 0xc4244df0e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4247a0508, 0xc4244f6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4244df0e0, 0xc4244f6000, 0x1000, 0x1000, 0xc424451b98, 0x403855, 0xc421b9a840)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421e9c660)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9c660, 0x1, 0x0, 0x0, 0x0, 0xc424014de0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4244df0e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 786 [select]:
net/http.(*persistConn).writeLoop(0xc425446a20)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 808 [select]:
net/http.(*persistConn).writeLoop(0xc425f7f440)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 801 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d6450, 0x72, 0xc424453960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4256fba98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4256fba98, 0xc425732000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4256fba80, 0xc425732000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4256fba80, 0xc425732000, 0x1000, 0x1000, 0xc426dcd388, 0xc42572ea20, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42571ee90, 0xc425732000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42572ea20, 0xc425732000, 0x1000, 0x1000, 0xc424453b98, 0x403855, 0xc421b9a960)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421c5cf00)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421c5cf00, 0x1, 0x0, 0x0, 0x0, 0xc4230192c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42572ea20)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 802 [select]:
net/http.(*persistConn).writeLoop(0xc42572ea20)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 756 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d6380, 0x72, 0xc42139a960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425856618, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425856618, 0xc4255f3000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425856600, 0xc4255f3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425856600, 0xc4255f3000, 0x1000, 0x1000, 0xc4232553a0, 0xc42587c480, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425511398, 0xc4255f3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42587c480, 0xc4255f3000, 0x1000, 0x1000, 0xc42139ab98, 0x403855, 0xc42140fe60)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421cd1500)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421cd1500, 0x1, 0x0, 0x0, 0x0, 0xc423e7f320, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42587c480)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 790 [select]:
net/http.(*persistConn).writeLoop(0xc4256a0a20)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 760 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d61e0, 0x72, 0xc425899960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425931998, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425931998, 0xc425891000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425931980, 0xc425891000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425931980, 0xc425891000, 0x1000, 0x1000, 0xc42773e210, 0xc4259557a0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4255113c8, 0xc425891000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4259557a0, 0xc425891000, 0x1000, 0x1000, 0xc425899b98, 0x403855, 0xc421298240)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4203cb2c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4203cb2c0, 0x1, 0x0, 0x0, 0x0, 0xc423e7f560, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4259557a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 805 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d6110, 0x72, 0xc424450960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425dc5198, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425dc5198, 0xc425df2000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425dc5180, 0xc425df2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425dc5180, 0xc425df2000, 0x1000, 0x1000, 0xc42773e218, 0xc425dee6c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425d45b38, 0xc425df2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425dee6c0, 0xc425df2000, 0x1000, 0x1000, 0xc424450b98, 0x403855, 0xc421b20d20)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f66c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f66c0, 0x1, 0x0, 0x0, 0x0, 0xc423e7f680, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425dee6c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 806 [select]:
net/http.(*persistConn).writeLoop(0xc425dee6c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 602 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5f70, 0x72, 0xc425df5960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261c4118, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261c4118, 0xc4261f0000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261c4100, 0xc4261f0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261c4100, 0xc4261f0000, 0x1000, 0x1000, 0xc427166088, 0xc424bf3200, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4261da010, 0xc4261f0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc424bf3200, 0xc4261f0000, 0x1000, 0x1000, 0xc425df5b98, 0x403855, 0xc4261e8180)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc424b6a000)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc424b6a000, 0x1, 0x0, 0x0, 0x0, 0xc42767fda0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc424bf3200)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 603 [select]:
net/http.(*persistConn).writeLoop(0xc424bf3200)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 812 [select]:
net/http.(*persistConn).writeLoop(0xc422d5b8c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 809 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5ea0, 0x72, 0xc425892960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc424292c18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc424292c18, 0xc4246a1000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc424292c00, 0xc4246a1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc424292c00, 0xc4246a1000, 0x1000, 0x1000, 0xc42773e230, 0xc424640120, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbbae8, 0xc4246a1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc424640120, 0xc4246a1000, 0x1000, 0x1000, 0xc425892b98, 0x403855, 0xc421b9b080)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f6060)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f6060, 0x1, 0x0, 0x0, 0x0, 0xc427170a80, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc424640120)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 815 [select]:
net/http.(*persistConn).writeLoop(0xc4254cd7a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 811 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5dd0, 0x72, 0xc425df4960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42472f618, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42472f618, 0xc4234ec000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42472f600, 0xc4234ec000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42472f600, 0xc4234ec000, 0x1000, 0x1000, 0xc42773e238, 0xc422d5b8c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbbaf0, 0xc4234ec000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc422d5b8c0, 0xc4234ec000, 0x1000, 0x1000, 0xc425df4b98, 0x403855, 0xc4200a4ea0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f6120)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f6120, 0x1, 0x0, 0x0, 0x0, 0xc427170ba0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc422d5b8c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 773 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5d00, 0x72, 0xc425016960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4245f2998, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4245f2998, 0xc42359b000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4245f2980, 0xc42359b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4245f2980, 0xc42359b000, 0x1000, 0x1000, 0xc42773e240, 0xc422f96360, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4261db070, 0xc42359b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc422f96360, 0xc42359b000, 0x1000, 0x1000, 0xc425016b98, 0x403855, 0xc4261e9b60)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42136d7a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42136d7a0, 0x1, 0x0, 0x0, 0x0, 0xc42767fec0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc422f96360)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 774 [select]:
net/http.(*persistConn).writeLoop(0xc422f96360)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 768 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5c30, 0x72, 0xc425018960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420fc0a98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420fc0a98, 0xc423798000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420fc0a80, 0xc423798000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420fc0a80, 0xc423798000, 0x1000, 0x1000, 0xc422f50578, 0xc4254cd560, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424b47740, 0xc423798000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4254cd560, 0xc423798000, 0x1000, 0x1000, 0xc425018b98, 0x403855, 0xc4261e9e60)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4214e31a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4214e31a0, 0x1, 0x0, 0x0, 0x0, 0xc427220000, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4254cd560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 817 [select]:
net/http.(*persistConn).writeLoop(0xc4254cd560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 833 [select]:
net/http.(*persistConn).writeLoop(0xc4254cd9e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 814 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5b60, 0x72, 0xc425019718)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420fc0f98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420fc0f98, 0xc428216000, 0x4000, 0x4000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420fc0f80, 0xc428216000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420fc0f80, 0xc428216000, 0x4000, 0x4000, 0xc4270d7b80, 0x5, 0xa)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4232547b8, 0xc428216000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc4256c7d70, 0x7f9a092f6150, 0xc4212bbf00, 0x5, 0xc4212bbf00, 0x8)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc421d35c00, 0x1b6e617, 0xc421d35d20, 0x456960)
	/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc421d35c00, 0xc423b67000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
net/http.(*persistConn).Read(0xc4254cd7a0, 0xc423b67000, 0x1000, 0x1000, 0xc425019b98, 0x403855, 0xc42128c240)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421c5d200)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421c5d200, 0x1, 0x0, 0x0, 0x0, 0xc4272403c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4254cd7a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 839 [select, 1 minutes]:
net/http.(*persistConn).writeLoop(0xc4266afd40)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 816 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d5a90, 0x72, 0xc425df8960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420fc1698, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420fc1698, 0xc42377d000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420fc1680, 0xc42377d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420fc1680, 0xc42377d000, 0x1000, 0x1000, 0xc42773e270, 0xc4254cd9e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4232547c8, 0xc42377d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4254cd9e0, 0xc42377d000, 0x1000, 0x1000, 0xc425df8b98, 0x403855, 0xc420184fc0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421c5d2c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421c5d2c0, 0x1, 0x0, 0x0, 0x0, 0xc423e7f7a0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4254cd9e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 836 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d59c0, 0x72, 0xc422c6f960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc426bee218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc426bee218, 0xc421803000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc426bee200, 0xc421803000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc426bee200, 0xc421803000, 0x1000, 0x1000, 0xc42773e2a0, 0xc4241fd200, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423254b38, 0xc421803000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4241fd200, 0xc421803000, 0x1000, 0x1000, 0xc422c6fb98, 0x403855, 0xc4201857a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421c5d620)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421c5d620, 0x1, 0x0, 0x0, 0x0, 0xc4272404e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4241fd200)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 837 [select]:
net/http.(*persistConn).writeLoop(0xc4241fd200)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 749 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a092d58f0, 0x72, 0xc426ca8960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc426c94c98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc426c94c98, 0xc42660b000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc426c94c80, 0xc42660b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc426c94c80, 0xc42660b000, 0x1000, 0x1000, 0xc42773e2a8, 0xc426ca65a0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426bcbfe8, 0xc42660b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc426ca65a0, 0xc42660b000, 0x1000, 0x1000, 0xc426ca8b98, 0x403855, 0xc426452fc0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4265d1d40)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4265d1d40, 0x1, 0x0, 0x0, 0x0, 0xc423e7f8c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc426ca65a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 842 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f9a092d5750, 0x72, 0xc421312d60)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc426bee298, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc426bee298, 0xc42645a400, 0x1, 0x1)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc426bee280, 0xc42645a4c1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc426bee280, 0xc42645a4c1, 0x1, 0x1, 0x0, 0x2, 0xc42025f110)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423254b48, 0xc42645a4c1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc424bab5e0, 0xc42645a4c1, 0x1, 0x1, 0x433508, 0x1b6e3e0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).backgroundRead(0xc42645a4b0)
	/usr/local/go/src/net/http/server.go:668 +0x5a
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:664 +0xce

goroutine 848 [select]:
net/http.(*persistConn).writeLoop(0xc425ffd7a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 845 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a0911a128, 0x72, 0xc426cac960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc426dc2818, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc426dc2818, 0xc426e0d000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc426dc2800, 0xc426e0d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc426dc2800, 0xc426e0d000, 0x1000, 0x1000, 0xc426dcd1a0, 0xc426de2120, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423255148, 0xc426e0d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc426de2120, 0xc426e0d000, 0x1000, 0x1000, 0xc426cacb98, 0x403855, 0xc426cb0780)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc426e32660)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc426e32660, 0x1, 0x0, 0x0, 0x0, 0xc426fc40c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc426de2120)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 846 [select, 1 minutes]:
net/http.(*persistConn).writeLoop(0xc426de2120)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 881 [semacquire]:
sync.runtime_Semacquire(0xc420038f7c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/storage/remote.(*Storage).Add(0xc420038f60, 0xc425fb8460, 0x5, 0x5, 0x1620a77ccf8, 0x3f19bbe037db0516, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/write.go:29 +0x5e
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc4258120c0, 0xc425fb8460, 0x5, 0x5, 0x1620a77ccf8, 0x3f19bbe037db0516, 0xc425fb8460, 0x5, 0x5)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:140 +0x131
github.com/prometheus/prometheus/scrape.(*timeLimitAppender).Add(0xc425864320, 0xc425fb8460, 0x5, 0x5, 0x1620a77ccf8, 0x3f19bbe037db0516, 0x1c36b01, 0xc420084060, 0x24f9)
	/go/src/github.com/prometheus/prometheus/scrape/target.go:243 +0x7b
github.com/prometheus/prometheus/scrape.(*scrapeLoop).append(0xc425f8f300, 0xc423bf4000, 0x124f9, 0x14f80, 0xbea0b8ff8bf3daf9, 0x5e34b5d12, 0x2883180, 0x20, 0xc421627124, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:880 +0x569
github.com/prometheus/prometheus/scrape.(*scrapeLoop).run(0xc425f8f300, 0x6fc23ac00, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:695 +0x682
created by github.com/prometheus/prometheus/scrape.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/scrape/scrape.go:303 +0x41f

goroutine 870 [select]:
net/http.(*persistConn).writeLoop(0xc4260bd8c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 823 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a0911a058, 0x72, 0xc42128b960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261c4298, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261c4298, 0xc423875000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261c4280, 0xc423875000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261c4280, 0xc423875000, 0x1000, 0x1000, 0xc424bbb8c8, 0xc4260d37a0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424b469d0, 0xc423875000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4260d37a0, 0xc423875000, 0x1000, 0x1000, 0xc42128bb98, 0x403855, 0xc42127a1e0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4214d20c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4214d20c0, 0x1, 0x0, 0x0, 0x0, 0xc421b9baa0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4260d37a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 824 [select]:
net/http.(*persistConn).writeLoop(0xc4260d37a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 827 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119eb8, 0x72, 0xc421890960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261cd598, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261cd598, 0xc425109000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261cd580, 0xc425109000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261cd580, 0xc425109000, 0x1000, 0x1000, 0xc42017e2f0, 0xc4260f6fc0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424b47078, 0xc425109000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4260f6fc0, 0xc425109000, 0x1000, 0x1000, 0xc421890b98, 0x403855, 0xc421d9a780)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4214d2cc0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4214d2cc0, 0x1, 0x0, 0x0, 0x0, 0xc421d9a6c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4260f6fc0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 828 [select]:
net/http.(*persistConn).writeLoop(0xc4260f6fc0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 866 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119de8, 0x72, 0xc4215b7960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261cd998, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261cd998, 0xc420fa3000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261cd980, 0xc420fa3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261cd980, 0xc420fa3000, 0x1000, 0x1000, 0xc425617440, 0xc4260f7200, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbb8d8, 0xc420fa3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4260f7200, 0xc420fa3000, 0x1000, 0x1000, 0xc4215b7b98, 0x403855, 0xc421efea20)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421da8c00)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421da8c00, 0x1, 0x0, 0x0, 0x0, 0xc421efe960, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4260f7200)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 867 [select]:
net/http.(*persistConn).writeLoop(0xc4260f7200)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 876 [select]:
net/http.(*persistConn).writeLoop(0xc4261166c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 869 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119d18, 0x72, 0xc421333960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261cdc18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261cdc18, 0xc42753f000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261cdc00, 0xc42753f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261cdc00, 0xc42753f000, 0x1000, 0x1000, 0xc425432ff0, 0xc4260bd8c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbb8f0, 0xc42753f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4260bd8c0, 0xc42753f000, 0x1000, 0x1000, 0xc421333b98, 0x403855, 0xc421efec00)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421da93e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421da93e0, 0x1, 0x0, 0x0, 0x0, 0xc421efeb40, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4260bd8c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 873 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119c48, 0x72, 0xc427db0960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261c4618, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261c4618, 0xc425f90000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261c4600, 0xc425f90000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261c4600, 0xc425f90000, 0x1000, 0x1000, 0xc4255113d0, 0xc4260d39e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc424bbb920, 0xc425f90000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc4260d39e0, 0xc425f90000, 0x1000, 0x1000, 0xc427db0b98, 0x403855, 0xc421b211a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421da9e00)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421da9e00, 0x1, 0x0, 0x0, 0x0, 0xc421eff320, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc4260d39e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 874 [select]:
net/http.(*persistConn).writeLoop(0xc4260d39e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 857 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119aa8, 0x72, 0xc426caf960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4261c5518, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4261c5518, 0xc4261b0000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4261c5500, 0xc4261b0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4261c5500, 0xc4261b0000, 0x1000, 0x1000, 0xc426dcd1c8, 0xc425ff4000, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425511d70, 0xc4261b0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425ff4000, 0xc4261b0000, 0x1000, 0x1000, 0xc426cafb98, 0x403855, 0xc421346840)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4204136e0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4204136e0, 0x1, 0x0, 0x0, 0x0, 0xc421346720, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425ff4000)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 888 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119838, 0x72, 0xc421393aa8)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f63018, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f63018, 0xc4215e6000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f63000, 0xc4215e6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f63000, 0xc4215e6000, 0x1000, 0x1000, 0xc421393bd0, 0x18, 0xc421393bc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423254d70, 0xc4215e6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4251f7ea0, 0xc4215e6000, 0x1000, 0x1000, 0x2883180, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).Read(0xc426145e00, 0xc4215e6000, 0x1000, 0x1000, 0x15c422a, 0xc426145d10, 0xbea0b94a8c4bd23d)
	/usr/local/go/src/net/http/server.go:764 +0xf8
bufio.(*Reader).fill(0xc421e9cf60)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421e9cf60, 0x4, 0x4bbd080c4a, 0x2883180, 0x0, 0x0, 0x2883180)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*conn).serve(0xc4251f7f40, 0x1c50c20, 0xc425f64ec0)
	/usr/local/go/src/net/http/server.go:1855 +0x7c3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1122 [select]:
net/http.(*persistConn).writeLoop(0xc425fcd0e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 795 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091195c8, 0x72, 0xc423546718)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f63c18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f63c18, 0xc423f80000, 0x2000, 0x2000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f63c00, 0xc423f80000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f63c00, 0xc423f80000, 0x2000, 0x2000, 0xc421b929bc, 0x4, 0xa)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426dcd190, 0xc423f80000, 0x2000, 0x2000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc425d5f530, 0x7f9a092f6150, 0xc42564a500, 0x5, 0xc42564a500, 0x30)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc42449e700, 0x1b6e617, 0xc42449e820, 0x456960)
	/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc42449e700, 0xc42548e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
net/http.(*persistConn).Read(0xc425ff5560, 0xc42548e000, 0x1000, 0x1000, 0xc423546b98, 0x403855, 0xc426cb0720)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42128e8a0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42128e8a0, 0x1, 0x0, 0x0, 0x0, 0xc426453aa0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425ff5560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1293 [semacquire]:
sync.runtime_SemacquireMutex(0xc42017d394, 0xc427555d00)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc42017d390)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc42017d390)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/scrape.(*Manager).Targets(0xc42017d360, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/manager.go:120 +0x68
github.com/prometheus/prometheus/web.(*Handler).targets(0xc42033ea00, 0x1c474e0, 0xc428f579a0, 0xc421f15c00)
	/go/src/github.com/prometheus/prometheus/web/web.go:613 +0x68
github.com/prometheus/prometheus/web.(*Handler).(github.com/prometheus/prometheus/web.targets)-fm(0x1c474e0, 0xc428f579a0, 0xc421f15c00)
	/go/src/github.com/prometheus/prometheus/web/web.go:219 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc429084900, 0xc421f15c00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc429084900, 0xc421f15c00)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc429084900, 0xc421f15a00, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc42016aac0, 0x1c47420, 0xc429084900, 0xc421f15a00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4201264a0, 0x1c47420, 0xc429084900, 0xc421f15a00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc429084900, 0xc421f15a00)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc421885880, 0xc421f15900)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc421885880, 0xc421f15900)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc421885880, 0xc421f15900)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc428f5d220, 0x1c50c20, 0xc4290885c0)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1204 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171cd0, 0x72, 0xc421332960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425cbe798, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425cbe798, 0xc424119000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425cbe780, 0xc424119000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425cbe780, 0xc424119000, 0x1000, 0x1000, 0xc426dcd2b0, 0xc425d1b560, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc422f501e0, 0xc424119000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425d1b560, 0xc424119000, 0x1000, 0x1000, 0xc421332b98, 0x403855, 0xc421cc7920)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42122d8c0)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42122d8c0, 0x1, 0x0, 0x0, 0x0, 0xc421cc7860, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d1b560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1177 [select]:
net/http.(*persistConn).writeLoop(0xc425d4fc20)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1173 [select]:
net/http.(*persistConn).writeLoop(0xc425d5b560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1152 [select]:
net/http.(*persistConn).writeLoop(0xc425d54120)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1118 [semacquire]:
sync.runtime_Semacquire(0xc420fc072c)
	/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*RWMutex).RLock(0xc420fc0720)
	/usr/local/go/src/sync/rwmutex.go:50 +0x49
github.com/prometheus/prometheus/scrape.(*Manager).Targets(0xc42017d360, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/manager.go:125 +0x236
github.com/prometheus/prometheus/web.(*Handler).targets(0xc42033ea00, 0x1c474e0, 0xc428c13bd0, 0xc421fbae00)
	/go/src/github.com/prometheus/prometheus/web/web.go:613 +0x68
github.com/prometheus/prometheus/web.(*Handler).(github.com/prometheus/prometheus/web.targets)-fm(0x1c474e0, 0xc428c13bd0, 0xc421fbae00)
	/go/src/github.com/prometheus/prometheus/web/web.go:219 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc428e3ac20, 0xc421fbae00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc428e3ac20, 0xc421fbae00)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc428e3ac20, 0xc421fbad00, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc42016aac0, 0x1c47420, 0xc428e3ac20, 0xc421fbad00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4201264a0, 0x1c47420, 0xc428e3ac20, 0xc421fbad00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc428e3ac20, 0xc421fbad00)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc421052380, 0xc421fbac00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc421052380, 0xc421fbac00)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc421052380, 0xc421fbac00)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc425fff2c0, 0x1c50c20, 0xc424796d80)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 796 [select]:
net/http.(*persistConn).writeLoop(0xc425ff5560)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1142 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091194f8, 0x72, 0xc423547960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425eeed18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425eeed18, 0xc423c68000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425eeed00, 0xc423c68000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425eeed00, 0xc423c68000, 0x1000, 0x1000, 0xc42773e040, 0xc425fb65a0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426dcd1c0, 0xc423c68000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425fb65a0, 0xc423c68000, 0x1000, 0x1000, 0xc423547b98, 0x403855, 0xc4264533e0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421244b40)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421244b40, 0x1, 0x0, 0x0, 0x0, 0xc4264532c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425fb65a0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1179 [select]:
net/http.(*persistConn).writeLoop(0xc425d48b40)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1094 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119428, 0x72, 0xc426df8960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425eef218, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425eef218, 0xc423d80000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425eef200, 0xc423d80000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425eef200, 0xc423d80000, 0x1000, 0x1000, 0xc42773e090, 0xc425fb39e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc425511e58, 0xc423d80000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425fb39e0, 0xc423d80000, 0x1000, 0x1000, 0xc426df8b98, 0x403855, 0xc4261eb4a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42122c240)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42122c240, 0x1, 0x0, 0x0, 0x0, 0xc4261eb3e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425fb39e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1154 [select]:
net/http.(*persistConn).writeLoop(0xc425f170e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1099 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09172280, 0x72, 0xc422c71960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425d9e318, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425d9e318, 0xc42302c000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425d9e300, 0xc42302c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425d9e300, 0xc42302c000, 0x1000, 0x1000, 0xc42773e120, 0xc425e2e6c0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc422f50100, 0xc42302c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425e2e6c0, 0xc42302c000, 0x1000, 0x1000, 0xc422c71b98, 0x403855, 0xc421cc6480)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42122ca20)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42122ca20, 0x1, 0x0, 0x0, 0x0, 0xc421cc63c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425e2e6c0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1172 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091720e0, 0x72, 0xc42188b960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f13f98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f13f98, 0xc423090000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f13f80, 0xc423090000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f13f80, 0xc423090000, 0x1000, 0x1000, 0xc4232552a8, 0xc425d5b560, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423255200, 0xc423090000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425d5b560, 0xc423090000, 0x1000, 0x1000, 0xc42188bb98, 0x403855, 0xc426453f20)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4215f6780)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215f6780, 0x1, 0x0, 0x0, 0x0, 0xc426fc5500, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d5b560)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1088 [select]:
net/http.(*persistConn).writeLoop(0xc425fc0c60)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1121 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119698, 0x72, 0xc427dad960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425fd3a18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425fd3a18, 0xc424350000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425fd3a00, 0xc424350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425fd3a00, 0xc424350000, 0x1000, 0x1000, 0xc425511e40, 0xc425fcd0e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc423255088, 0xc424350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425fcd0e0, 0xc424350000, 0x1000, 0x1000, 0xc427dadb98, 0x403855, 0xc421b214a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4205c8660)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4205c8660, 0x1, 0x0, 0x0, 0x0, 0xc421b213e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425fcd0e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1151 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091721b0, 0x72, 0xc423548718)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f13c98, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f13c98, 0xc4244e5000, 0x400, 0x400)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f13c80, 0xc4244e5000, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f13c80, 0xc4244e5000, 0x400, 0x400, 0x1, 0xc422fff500, 0xc423548880)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4232551e0, 0xc4244e5000, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc4258dd260, 0x7f9a092f6150, 0xc4252ad180, 0x5, 0xc4252ad180, 0x400)
	/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc422fb6a80, 0x1b6e617, 0xc422fb6ba0, 0x4b492787ea130a8d)
	/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc422fb6a80, 0xc423f4f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
net/http.(*persistConn).Read(0xc425d54120, 0xc423f4f000, 0x1000, 0x1000, 0x403666, 0xc423548b98, 0x429f69)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421245c20)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421245c20, 0x1, 0xc426453c20, 0xc42434cc80, 0xc42128efc0, 0x60, 0x60)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425d54120)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1095 [select]:
net/http.(*persistConn).writeLoop(0xc425fb39e0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1100 [select]:
net/http.(*persistConn).writeLoop(0xc425e2e6c0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1153 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119358, 0x72, 0xc4277b6960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425ebb618, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425ebb618, 0xc421b7e000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425ebb600, 0xc421b7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425ebb600, 0xc421b7e000, 0x1000, 0x1000, 0xc425511eb0, 0xc425f170e0, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42773e0b0, 0xc421b7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425f170e0, 0xc421b7e000, 0x1000, 0x1000, 0xc4277b6b98, 0x403855, 0xc426cb17a0)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc42128ff20)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc42128ff20, 0x1, 0x0, 0x0, 0x0, 0xc426cb16e0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425f170e0)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1143 [select]:
net/http.(*persistConn).writeLoop(0xc425fb65a0)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1345 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171720, 0x72, 0xc4292cbaa8)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc424826698, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc424826698, 0xc427599000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc424826680, 0xc427599000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc424826680, 0xc427599000, 0x1000, 0x1000, 0xc4292cbbd0, 0x18, 0xc4292cbbc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42773e358, 0xc427599000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc428f5d2c0, 0xc427599000, 0x1000, 0x1000, 0x2883180, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).Read(0xc4244f3ce0, 0xc427599000, 0x1000, 0x1000, 0x15c422a, 0xc4244f3b60, 0xbea0b964c6b93d2c)
	/usr/local/go/src/net/http/server.go:764 +0xf8
bufio.(*Reader).fill(0xc4215d2960)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4215d2960, 0x4, 0x6429f25134, 0x2883180, 0x0, 0x0, 0x2883180)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*conn).serve(0xc428f5d360, 0x1c50c20, 0xc4292b8940)
	/usr/local/go/src/net/http/server.go:1855 +0x7c3
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1294 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09171580, 0x72, 0xc423cbb560)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4202e6a18, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4202e6a18, 0xc424bf4c00, 0x1, 0x1)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4202e6a00, 0xc424bf4c11, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4202e6a00, 0xc424bf4c11, 0x1, 0x1, 0x0, 0x2, 0xc42025f110)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42773e348, 0xc424bf4c11, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc428f5d180, 0xc424bf4c11, 0x1, 0x1, 0x3, 0x6020103, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).backgroundRead(0xc424bf4c00)
	/usr/local/go/src/net/http/server.go:668 +0x5a
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:664 +0xce

goroutine 1148 [select]:
net/http.(*persistConn).writeLoop(0xc425f7ed80)
	/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 1147 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a09119288, 0x72, 0xc4241bc960)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc425f13298, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc425f13298, 0xc423d9d000, 0x1000, 0x1000)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc425f13280, 0xc423d9d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc425f13280, 0xc423d9d000, 0x1000, 0x1000, 0xc4232551f0, 0xc425f7ed80, 0x4)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc426dcd240, 0xc423d9d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc425f7ed80, 0xc423d9d000, 0x1000, 0x1000, 0xc4241bcb98, 0x403855, 0xc426cb1800)
	/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc421245560)
	/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc421245560, 0x1, 0x0, 0x0, 0x0, 0xc4264538c0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc425f7ed80)
	/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 1318 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ce0, 0xc422d11cb0, 0x8000000000000000, 0x7fffffffffffffff, 0x1c4f0a0, 0xc420e19350, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ce0, 0xc422d11cb0, 0x8000000000000000, 0x7fffffffffffffff, 0x8, 0xc420392120, 0x2874a30, 0xc4202d0240)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/web/api/v1.(*API).labelValues(0xc420573ab0, 0xc421d3ae00, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:341 +0x142
github.com/prometheus/prometheus/web/api/v1.(*API).(github.com/prometheus/prometheus/web/api/v1.labelValues)-fm(0xc421d3ae00, 0xc429057f60, 0xc4283d0cd0, 0x3)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:179 +0x34
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1c45ca0, 0xc429057f60, 0xc421d3ae00)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:159 +0x55
net/http.HandlerFunc.ServeHTTP(0xc4202ff110, 0x1c45ca0, 0xc429057f60, 0xc421d3ae00)
	/usr/local/go/src/net/http/server.go:1947 +0x44
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1c3b7e0, 0xc4202ff110, 0x1c474e0, 0xc4283d0cd0, 0xc421d3ae00)
	/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x7c
net/http.(Handler).ServeHTTP-fm(0x1c474e0, 0xc4283d0cd0, 0xc421d3ae00)
	/usr/local/go/src/net/http/h2_bundle.go:5475 +0x4d
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc429057f20, 0xc421d3ae00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc429057f20, 0xc421d3ae00)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc429057f20, 0xc421d3ad00, 0xc429057f40, 0x1, 0x1)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc4206f0540, 0x1c47420, 0xc429057f20, 0xc421d3ad00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc420126740, 0x1c47420, 0xc429057f20, 0xc421d3ad00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.StripPrefix.func1(0x1c47420, 0xc429057f20, 0xc421d3ac00)
	/usr/local/go/src/net/http/server.go:1986 +0x19a
net/http.HandlerFunc.ServeHTTP(0xc420f1e2d0, 0x1c47420, 0xc429057f20, 0xc421d3ac00)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc429057f20, 0xc421d3ac00)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc42194aee0, 0xc421d3ab00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc42194aee0, 0xc421d3ab00)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc42194aee0, 0xc421d3ab00)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc425fff400, 0x1c50c20, 0xc42906e3c0)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1114 [semacquire]:
sync.runtime_SemacquireMutex(0xc42017d394, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc42017d390)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc42017d390)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/scrape.(*Manager).Targets(0xc42017d360, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/scrape/manager.go:120 +0x68
github.com/prometheus/prometheus/web.(*Handler).targets(0xc42033ea00, 0x1c474e0, 0xc426d547d0, 0xc421d43000)
	/go/src/github.com/prometheus/prometheus/web/web.go:613 +0x68
github.com/prometheus/prometheus/web.(*Handler).(github.com/prometheus/prometheus/web.targets)-fm(0x1c474e0, 0xc426d547d0, 0xc421d43000)
	/go/src/github.com/prometheus/prometheus/web/web.go:219 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc428f87e40, 0xc421d43000)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc428f87e40, 0xc421d43000)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc428f87e40, 0xc421d42f00, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc42016aac0, 0x1c47420, 0xc428f87e40, 0xc421d42f00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc4201264a0, 0x1c47420, 0xc428f87e40, 0xc421d42f00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc428f87e40, 0xc421d42f00)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc4214bd880, 0xc421d42e00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc4214bd880, 0xc421d42e00)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc4214bd880, 0xc421d42e00)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc425fff220, 0x1c50c20, 0xc424796900)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1319 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091714b0, 0x72, 0xc420066560)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4203dc398, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4203dc398, 0xc422d11900, 0x1, 0x1)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4203dc380, 0xc422d119f1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4203dc380, 0xc422d119f1, 0x1, 0x1, 0xc428e39a30, 0xc420066750, 0x8)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc422f50eb8, 0xc422d119f1, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc425fff360, 0xc422d119f1, 0x1, 0x1, 0x0, 0x8020102, 0x433508)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).backgroundRead(0xc422d119e0)
	/usr/local/go/src/net/http/server.go:668 +0x5a
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:664 +0xce

goroutine 1302 [semacquire]:
sync.runtime_SemacquireMutex(0xc420038f74, 0x40e800)
	/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/mutex.go:134 +0x108
sync.(*RWMutex).Lock(0xc420038f70)
	/usr/local/go/src/sync/rwmutex.go:93 +0x2d
github.com/prometheus/prometheus/storage/remote.(*Storage).Querier(0xc420038f60, 0x1c50ca0, 0xc42136d9e0, 0x1620a790f29, 0x1620a790f29, 0x1c4f0a0, 0xc428dabf40, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/storage/remote/storage.go:124 +0x4e
github.com/prometheus/prometheus/storage.(*fanout).Querier(0xc42016a380, 0x1c50ca0, 0xc42136d9e0, 0x1620a790f29, 0x1620a790f29, 0x1917892573, 0x2883180, 0x289e1393285baf50, 0xc4285baef8)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:77 +0x1cc
github.com/prometheus/prometheus/promql.(*Engine).populateIterators(0xc4203c2900, 0x1c50ca0, 0xc42136d9e0, 0x7f9a0946e1d8, 0xc42016a380, 0xc428f57770, 0xc4285bafe0, 0x441c67, 0x8000, 0x1744dc0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:494 +0x2b7
github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt(0xc4203c2900, 0x1c50ca0, 0xc42136d9e0, 0xc428f577c0, 0xc428f57770, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:346 +0xf1
github.com/prometheus/prometheus/promql.(*Engine).exec(0xc4203c2900, 0x1c50ca0, 0xc42136d9e0, 0xc428f577c0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:327 +0x35f
github.com/prometheus/prometheus/promql.(*query).Exec(0xc428f577c0, 0x1c50c20, 0xc429088540, 0xc428f1ed49)
	/go/src/github.com/prometheus/prometheus/promql/engine.go:130 +0xb8
github.com/prometheus/prometheus/web/api/v1.(*API).query(0xc420573ab0, 0xc421f15800, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:236 +0x2b2
github.com/prometheus/prometheus/web/api/v1.(*API).(github.com/prometheus/prometheus/web/api/v1.query)-fm(0xc421f15800, 0xc4290847c0, 0xc428f57720, 0x3)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:174 +0x34
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1c45ca0, 0xc4290847c0, 0xc421f15800)
	/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:159 +0x55
net/http.HandlerFunc.ServeHTTP(0xc4202fe460, 0x1c45ca0, 0xc4290847c0, 0xc421f15800)
	/usr/local/go/src/net/http/server.go:1947 +0x44
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1c3b7e0, 0xc4202fe460, 0x1c474e0, 0xc428f57720, 0xc421f15800)
	/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x7c
net/http.(Handler).ServeHTTP-fm(0x1c474e0, 0xc428f57720, 0xc421f15800)
	/usr/local/go/src/net/http/h2_bundle.go:5475 +0x4d
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x1c47420, 0xc4290847a0, 0xc421f15800)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:309 +0x229
github.com/prometheus/prometheus/web.(*Handler).testReady.func1(0x1c47420, 0xc4290847a0, 0xc421f15800)
	/go/src/github.com/prometheus/prometheus/web/web.go:357 +0x55
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x1c47420, 0xc4290847a0, 0xc421f15700, 0x0, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:50 +0x26a
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc4206f0540, 0x1c47420, 0xc4290847a0, 0xc421f15700)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x6d1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc420126740, 0x1c47420, 0xc4290847a0, 0xc421f15700)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:88 +0x4c
net/http.StripPrefix.func1(0x1c47420, 0xc4290847a0, 0xc421f15400)
	/usr/local/go/src/net/http/server.go:1986 +0x19a
net/http.HandlerFunc.ServeHTTP(0xc420f1e2d0, 0x1c47420, 0xc4290847a0, 0xc421f15400)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.(*ServeMux).ServeHTTP(0xc42025fb90, 0x1c47420, 0xc4290847a0, 0xc421f15400)
	/usr/local/go/src/net/http/server.go:2337 +0x130
github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp.Middleware.func2(0x1c4ed60, 0xc4218856c0, 0xc421f15300)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:74 +0x3f3
net/http.HandlerFunc.ServeHTTP(0xc420f1e360, 0x1c4ed60, 0xc4218856c0, 0xc421f15300)
	/usr/local/go/src/net/http/server.go:1947 +0x44
net/http.serverHandler.ServeHTTP(0xc42053e820, 0x1c4ed60, 0xc4218856c0, 0xc421f15300)
	/usr/local/go/src/net/http/server.go:2694 +0xbc
net/http.(*conn).serve(0xc425fe34a0, 0x1c50c20, 0xc424ace340)
	/usr/local/go/src/net/http/server.go:1830 +0x651
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:2795 +0x27b

goroutine 1289 [IO wait]:
internal/poll.runtime_pollWait(0x7f9a091717f0, 0x72, 0xc424ac0560)
	/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42579f418, 0x72, 0xffffffffffffff00, 0x1c3bd00, 0x27b7638)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42579f418, 0xc42228e900, 0x1, 0x1)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc42579f400, 0xc42228e911, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc42579f400, 0xc42228e911, 0x1, 0x1, 0x0, 0x1c39ac0, 0x28a3fc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4232553b8, 0xc42228e911, 0x1, 0x1, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:176 +0x6a
github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc425fe3400, 0xc42228e911, 0x1, 0x1, 0xc428f576d0, 0x1c39c01, 0xc428f576d0)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x119
net/http.(*connReader).backgroundRead(0xc42228e900)
	/usr/local/go/src/net/http/server.go:668 +0x5a
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:664 +0xce