Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save roidelapluie/95a71d16b7c10be8bd8eee01694e1fe2 to your computer and use it in GitHub Desktop.
Save roidelapluie/95a71d16b7c10be8bd8eee01694e1fe2 to your computer and use it in GitHub Desktop.
goroutine profile: total 7614
2503 @ 0x431340 0x440e4b 0x707ce3 0x45f181
# 0x707ce2 net/http.(*persistConn).writeLoop+0x122 /usr/local/go/src/net/http/transport.go:2210
2502 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x705595 0x5b9c33 0x5b9d9f 0x706246 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x705594 net/http.(*persistConn).Read+0x74 /usr/local/go/src/net/http/transport.go:1758
# 0x5b9c32 bufio.(*Reader).fill+0x102 /usr/local/go/src/bufio/bufio.go:100
# 0x5b9d9e bufio.(*Reader).Peek+0x4e /usr/local/go/src/bufio/bufio.go:138
# 0x706245 net/http.(*persistConn).readLoop+0x1d5 /usr/local/go/src/net/http/transport.go:1911
2499 @ 0x431340 0x440e4b 0x1d7eece 0x45f181
# 0x1d7eecd github.com/prometheus/prometheus/scrape.(*scrapeLoop).run+0x87d /go/src/github.com/prometheus/prometheus/scrape/scrape.go:987
6 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1cdfa8d 0x1cdf8e7 0x1ce89d7 0x1ce7c13 0x1cea81c 0x1ce62aa 0x1ce6250 0x1ce4d95 0x1d2629f 0x1c73ccc 0x1c73a19 0x1c73b63 0x1c73b63 0x1c73b63 0x1d0bf0a 0x1d0be10 0x1d0a96a 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1cdfa8c sync.(*RWMutex).RLock+0x48c /usr/local/go/src/sync/rwmutex.go:50
# 0x1cdf8e6 github.com/prometheus/prometheus/tsdb.(*headIndexReader).Postings+0x2e6 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1429
# 0x1ce89d6 github.com/prometheus/prometheus/tsdb.postingsForMatcher+0x3e6 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:424
# 0x1ce7c12 github.com/prometheus/prometheus/tsdb.PostingsForMatchers+0x292 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:373
# 0x1cea81b github.com/prometheus/prometheus/tsdb.lookupChunkSeries+0x8b /go/src/github.com/prometheus/prometheus/tsdb/querier.go:711
# 0x1ce62a9 github.com/prometheus/prometheus/tsdb.LookupChunkSeries+0x89 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:698
# 0x1ce624f github.com/prometheus/prometheus/tsdb.(*blockQuerier).Select+0x2f /go/src/github.com/prometheus/prometheus/tsdb/querier.go:199
# 0x1ce4d94 github.com/prometheus/prometheus/tsdb.(*querier).Select+0x114 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:94
# 0x1d2629e github.com/prometheus/prometheus/promql.(*Engine).populateSeries.func1+0x3ce /go/src/github.com/prometheus/prometheus/promql/engine.go:692
# 0x1c73ccb github.com/prometheus/prometheus/promql/parser.inspector.Visit+0x5b /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:293
# 0x1c73a18 github.com/prometheus/prometheus/promql/parser.Walk+0x78 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:275
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1d0bf09 github.com/prometheus/prometheus/promql/parser.Inspect+0x169 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:305
# 0x1d0be0f github.com/prometheus/prometheus/promql.(*Engine).populateSeries+0x6f /go/src/github.com/prometheus/prometheus/promql/engine.go:655
# 0x1d0a969 github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x319 /go/src/github.com/prometheus/prometheus/promql/engine.go:513
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
5 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x1e84f5c 0x1e862ef 0x6e4968 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x1e84f5b github.com/soheilhy/cmux.(*bufferedReader).Read+0x12b /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/buffer.go:53
# 0x1e862ee github.com/soheilhy/cmux.(*MuxConn).Read+0x4e /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:260
# 0x6e4967 net/http.(*connReader).backgroundRead+0x57 /usr/local/go/src/net/http/server.go:677
4 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1625dd9 0x1565fae 0x1566069 0x45f181
# 0x1625dd8 k8s.io/client-go/tools/cache.(*sharedProcessor).run+0x48 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:529
# 0x1565fad k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1+0x2d /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54
# 0x1566068 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1+0x58 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71
4 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x162a7a4 0x45f181
# 0x162a7a3 k8s.io/client-go/tools/cache.(*controller).Run.func1+0x33 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:104
4 @ 0x431340 0x440e4b 0x16226fc 0x1621fb1 0x162a803 0x15660ee 0x1565588 0x16215c8 0x1621568 0x1565fae 0x1566069 0x45f181
# 0x16226fb k8s.io/client-go/tools/cache.(*Reflector).watchHandler+0x1eb /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:354
# 0x1621fb0 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch+0x8b0 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:320
# 0x162a802 k8s.io/client-go/tools/cache.(*Reflector).Run.func1+0x32 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:152
# 0x15660ed k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1+0x5d /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152
# 0x1565587 k8s.io/apimachinery/pkg/util/wait.JitterUntil+0xf7 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153
# 0x16215c7 k8s.io/apimachinery/pkg/util/wait.Until+0x197 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
# 0x1621567 k8s.io/client-go/tools/cache.(*Reflector).Run+0x137 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:151
# 0x1565fad k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1+0x2d /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:54
# 0x1566068 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1+0x58 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71
4 @ 0x431340 0x440e4b 0x162b922 0x45f181
# 0x162b921 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2+0x171 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/reflector.go:260
4 @ 0x431340 0x440e4b 0x7089b0 0x6fd816 0x6e2e45 0x6a1163 0x6a0b7a 0x6a226e 0x1d7be07 0x1d7bd29 0x1d7eb0e 0x45f181
# 0x7089af net/http.(*persistConn).roundTrip+0x76f /usr/local/go/src/net/http/transport.go:2431
# 0x6fd815 net/http.(*Transport).roundTrip+0x9b5 /usr/local/go/src/net/http/transport.go:535
# 0x6e2e44 net/http.(*Transport).RoundTrip+0x34 /usr/local/go/src/net/http/roundtrip.go:17
# 0x6a1162 net/http.send+0x442 /usr/local/go/src/net/http/client.go:250
# 0x6a0b79 net/http.(*Client).send+0xf9 /usr/local/go/src/net/http/client.go:174
# 0x6a226d net/http.(*Client).do+0x3cd /usr/local/go/src/net/http/client.go:641
# 0x1d7be06 net/http.(*Client).Do+0x136 /usr/local/go/src/net/http/client.go:509
# 0x1d7bd28 github.com/prometheus/prometheus/scrape.(*targetScraper).scrape+0x58 /go/src/github.com/prometheus/prometheus/scrape/scrape.go:549
# 0x1d7eb0d github.com/prometheus/prometheus/scrape.(*scrapeLoop).run+0x4bd /go/src/github.com/prometheus/prometheus/scrape/scrape.go:946
4 @ 0x431340 0x4429b8 0x44298e 0x46b72d 0x161ea36 0x161c860 0x15660ee 0x1565588 0x161c60d 0x161c5af 0x162445f 0x45f181
# 0x44298d sync.runtime_notifyListWait+0xcd /usr/local/go/src/runtime/sema.go:510
# 0x46b72c sync.(*Cond).Wait+0x9c /usr/local/go/src/sync/cond.go:56
# 0x161ea35 k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop+0xa5 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:409
# 0x161c85f k8s.io/client-go/tools/cache.(*controller).processLoop+0x3f /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:153
# 0x15660ed k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1+0x5d /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152
# 0x1565587 k8s.io/apimachinery/pkg/util/wait.JitterUntil+0xf7 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153
# 0x161c60c k8s.io/apimachinery/pkg/util/wait.Until+0x2fc /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
# 0x161c5ae k8s.io/client-go/tools/cache.(*controller).Run+0x29e /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/controller.go:125
# 0x162445e k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run+0x47e /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:331
4 @ 0x431340 0x4429b8 0x44298e 0x46b72d 0xed0fa6 0xedd49c 0x75e11b 0x75deac 0x75d979 0x15e60f1 0x1575a09 0x1575f5c 0x12f2e85 0x45f181
# 0x44298d sync.runtime_notifyListWait+0xcd /usr/local/go/src/runtime/sema.go:510
# 0x46b72c sync.(*Cond).Wait+0x9c /usr/local/go/src/sync/cond.go:56
# 0xed0fa5 golang.org/x/net/http2.(*pipe).Read+0xa5 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/pipe.go:65
# 0xedd49b golang.org/x/net/http2.transportResponseBody.Read+0xab /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:2019
# 0x75e11a encoding/json.(*Decoder).refill+0xea /usr/local/go/src/encoding/json/stream.go:161
# 0x75deab encoding/json.(*Decoder).readValue+0x1db /usr/local/go/src/encoding/json/stream.go:136
# 0x75d978 encoding/json.(*Decoder).Decode+0x78 /usr/local/go/src/encoding/json/stream.go:63
# 0x15e60f0 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read+0x1a0 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/framer/framer.go:150
# 0x1575a08 k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode+0x88 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go:77
# 0x1575f5b k8s.io/client-go/rest/watch.(*Decoder).Decode+0x7b /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/rest/watch/decoder.go:49
# 0x12f2e84 k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive+0x174 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:104
3 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x162c024 0x1565931 0x162c2a9 0x15660ee 0x1565588 0x16268eb 0x1626890 0x1566069 0x45f181
# 0x162c023 k8s.io/client-go/tools/cache.(*processorListener).run.func1.1+0x53 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:653
# 0x1565930 k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff+0x50 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:292
# 0x162c2a8 k8s.io/client-go/tools/cache.(*processorListener).run.func1+0x78 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:652
# 0x15660ed k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1+0x5d /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:152
# 0x1565587 k8s.io/apimachinery/pkg/util/wait.JitterUntil+0xf7 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:153
# 0x16268ea k8s.io/apimachinery/pkg/util/wait.Until+0x9a /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
# 0x162688f k8s.io/client-go/tools/cache.(*processorListener).run+0x3f /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:650
# 0x1566068 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1+0x58 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71
3 @ 0x431340 0x440e4b 0x162669d 0x1566069 0x45f181
# 0x162669c k8s.io/client-go/tools/cache.(*processorListener).pop+0x18c /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/tools/cache/shared_informer.go:621
# 0x1566068 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1+0x58 /go/src/github.com/prometheus/prometheus/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:71
3 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1cdfa8d 0x1cdf8e7 0x1ce8aa0 0x1ce7c13 0x1cea81c 0x1ce62aa 0x1ce6250 0x1ce4d95 0x1d2629f 0x1c73ccc 0x1c73a19 0x1c73b63 0x1c73b63 0x1c73b63 0x1d0bf0a 0x1d0be10 0x1d0a96a 0x1d0a106 0x1d07cf4 0x1e8e84a 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1cdfa8c sync.(*RWMutex).RLock+0x48c /usr/local/go/src/sync/rwmutex.go:50
# 0x1cdf8e6 github.com/prometheus/prometheus/tsdb.(*headIndexReader).Postings+0x2e6 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1429
# 0x1ce8a9f github.com/prometheus/prometheus/tsdb.postingsForMatcher+0x4af /go/src/github.com/prometheus/prometheus/tsdb/querier.go:416
# 0x1ce7c12 github.com/prometheus/prometheus/tsdb.PostingsForMatchers+0x292 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:373
# 0x1cea81b github.com/prometheus/prometheus/tsdb.lookupChunkSeries+0x8b /go/src/github.com/prometheus/prometheus/tsdb/querier.go:711
# 0x1ce62a9 github.com/prometheus/prometheus/tsdb.LookupChunkSeries+0x89 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:698
# 0x1ce624f github.com/prometheus/prometheus/tsdb.(*blockQuerier).Select+0x2f /go/src/github.com/prometheus/prometheus/tsdb/querier.go:199
# 0x1ce4d94 github.com/prometheus/prometheus/tsdb.(*querier).Select+0x114 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:94
# 0x1d2629e github.com/prometheus/prometheus/promql.(*Engine).populateSeries.func1+0x3ce /go/src/github.com/prometheus/prometheus/promql/engine.go:692
# 0x1c73ccb github.com/prometheus/prometheus/promql/parser.inspector.Visit+0x5b /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:293
# 0x1c73a18 github.com/prometheus/prometheus/promql/parser.Walk+0x78 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:275
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1d0bf09 github.com/prometheus/prometheus/promql/parser.Inspect+0x169 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:305
# 0x1d0be0f github.com/prometheus/prometheus/promql.(*Engine).populateSeries+0x6f /go/src/github.com/prometheus/prometheus/promql/engine.go:655
# 0x1d0a969 github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x319 /go/src/github.com/prometheus/prometheus/promql/engine.go:513
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8e849 github.com/prometheus/prometheus/web/api/v1.(*API).queryRange+0xb59 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:418
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
2 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x122cfa0 0x45f181
# 0x122cf9f k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop+0xdf /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/util/workqueue/queue.go:198
2 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x1e84f5c 0x1e862ef 0x6e4db4 0x5b9c33 0x5b9d9f 0x6ea940 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x1e84f5b github.com/soheilhy/cmux.(*bufferedReader).Read+0x12b /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/buffer.go:53
# 0x1e862ee github.com/soheilhy/cmux.(*MuxConn).Read+0x4e /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:260
# 0x6e4db3 net/http.(*connReader).Read+0xf3 /usr/local/go/src/net/http/server.go:785
# 0x5b9c32 bufio.(*Reader).fill+0x102 /usr/local/go/src/bufio/bufio.go:100
# 0x5b9d9e bufio.(*Reader).Peek+0x4e /usr/local/go/src/bufio/bufio.go:138
# 0x6ea93f net/http.(*conn).serve+0x9df /usr/local/go/src/net/http/server.go:1915
2 @ 0x431340 0x440e4b 0x1b82553 0x45f181
# 0x1b82552 github.com/prometheus/prometheus/discovery.(*Manager).updater+0x1b2 /go/src/github.com/prometheus/prometheus/discovery/manager.go:236
2 @ 0x431340 0x440e4b 0x1b82913 0x45f181
# 0x1b82912 github.com/prometheus/prometheus/discovery.(*Manager).sender+0x122 /go/src/github.com/prometheus/prometheus/discovery/manager.go:263
2 @ 0x431340 0x440e4b 0x1d23454 0x1d0a3a5 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea 0x1ec7916 0x1e845a6 0x1e7fc6e 0x1e8415c 0x70ee6a 0x6eb9e4 0x6ed9bd 0x1e68e1b 0x6eb9e4 0x1ec5613 0x6eb9e4 0x6eef34 0x6ea7d5 0x45f181
# 0x1d23453 github.com/prometheus/prometheus/promql.ActiveQueryTracker.Insert+0xd3 /go/src/github.com/prometheus/prometheus/promql/query_logger.go:185
# 0x1d0a3a4 github.com/prometheus/prometheus/promql.(*Engine).exec+0x764 /go/src/github.com/prometheus/prometheus/promql/engine.go:462
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
# 0x1ec7915 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1+0x1c5 /go/src/github.com/prometheus/prometheus/web/web.go:1122
# 0x1e845a5 github.com/prometheus/common/route.(*Router).handle.func1+0x295 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83
# 0x1e7fc6d github.com/julienschmidt/httprouter.(*Router).ServeHTTP+0x95d /go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:334
# 0x1e8415b github.com/prometheus/common/route.(*Router).ServeHTTP+0x4b /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:121
# 0x70ee69 net/http.StripPrefix.func1+0x189 /usr/local/go/src/net/http/server.go:2046
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x6ed9bc net/http.(*ServeMux).ServeHTTP+0x1bc /usr/local/go/src/net/http/server.go:2387
# 0x1e68e1a github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5+0x5fa /go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:140
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1ec5612 github.com/prometheus/prometheus/web.withStackTracer.func1+0xb2 /go/src/github.com/prometheus/prometheus/web/web.go:103
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x6eef33 net/http.serverHandler.ServeHTTP+0xa3 /usr/local/go/src/net/http/server.go:2802
# 0x6ea7d4 net/http.(*conn).serve+0x874 /usr/local/go/src/net/http/server.go:1890
2 @ 0x431340 0x440e4b 0x1d23454 0x1d0a3a5 0x1d07cf4 0x1e8e84a 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea 0x1ec7916 0x1e845a6 0x1e7fc6e 0x1e8415c 0x70ee6a 0x6eb9e4 0x6ed9bd 0x1e68e1b 0x6eb9e4 0x1ec5613 0x6eb9e4 0x6eef34 0x6ea7d5 0x45f181
# 0x1d23453 github.com/prometheus/prometheus/promql.ActiveQueryTracker.Insert+0xd3 /go/src/github.com/prometheus/prometheus/promql/query_logger.go:185
# 0x1d0a3a4 github.com/prometheus/prometheus/promql.(*Engine).exec+0x764 /go/src/github.com/prometheus/prometheus/promql/engine.go:462
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8e849 github.com/prometheus/prometheus/web/api/v1.(*API).queryRange+0xb59 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:418
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
# 0x1ec7915 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1+0x1c5 /go/src/github.com/prometheus/prometheus/web/web.go:1122
# 0x1e845a5 github.com/prometheus/common/route.(*Router).handle.func1+0x295 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83
# 0x1e7fc6d github.com/julienschmidt/httprouter.(*Router).ServeHTTP+0x95d /go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:334
# 0x1e8415b github.com/prometheus/common/route.(*Router).ServeHTTP+0x4b /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:121
# 0x70ee69 net/http.StripPrefix.func1+0x189 /usr/local/go/src/net/http/server.go:2046
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x6ed9bc net/http.(*ServeMux).ServeHTTP+0x1bc /usr/local/go/src/net/http/server.go:2387
# 0x1e68e1a github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5+0x5fa /go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:140
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1ec5612 github.com/prometheus/prometheus/web.withStackTracer.func1+0xb2 /go/src/github.com/prometheus/prometheus/web/web.go:103
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x6eef33 net/http.serverHandler.ServeHTTP+0xa3 /usr/local/go/src/net/http/server.go:2802
# 0x6ea7d4 net/http.(*conn).serve+0x874 /usr/local/go/src/net/http/server.go:1890
2 @ 0x431340 0x4419b0 0x44199b 0x441602 0x46f024 0x1b1bf9e 0x45f181
# 0x441601 sync.runtime_Semacquire+0x41 /usr/local/go/src/runtime/sema.go:56
# 0x46f023 sync.(*WaitGroup).Wait+0x63 /usr/local/go/src/sync/waitgroup.go:130
# 0x1b1bf9d github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run+0x5ad /go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:465
2 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1cdfa8d 0x1cdf8e7 0x1ce89d7 0x1ce7c13 0x1cea81c 0x1ce62aa 0x1ce6250 0x1ce4d95 0x1d2629f 0x1c73ccc 0x1c73a19 0x1c73b63 0x1c73b63 0x1c73b63 0x1d0bf0a 0x1d0be10 0x1d0a96a 0x1d0a106 0x1d07cf4 0x1e8e84a 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1cdfa8c sync.(*RWMutex).RLock+0x48c /usr/local/go/src/sync/rwmutex.go:50
# 0x1cdf8e6 github.com/prometheus/prometheus/tsdb.(*headIndexReader).Postings+0x2e6 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1429
# 0x1ce89d6 github.com/prometheus/prometheus/tsdb.postingsForMatcher+0x3e6 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:424
# 0x1ce7c12 github.com/prometheus/prometheus/tsdb.PostingsForMatchers+0x292 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:373
# 0x1cea81b github.com/prometheus/prometheus/tsdb.lookupChunkSeries+0x8b /go/src/github.com/prometheus/prometheus/tsdb/querier.go:711
# 0x1ce62a9 github.com/prometheus/prometheus/tsdb.LookupChunkSeries+0x89 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:698
# 0x1ce624f github.com/prometheus/prometheus/tsdb.(*blockQuerier).Select+0x2f /go/src/github.com/prometheus/prometheus/tsdb/querier.go:199
# 0x1ce4d94 github.com/prometheus/prometheus/tsdb.(*querier).Select+0x114 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:94
# 0x1d2629e github.com/prometheus/prometheus/promql.(*Engine).populateSeries.func1+0x3ce /go/src/github.com/prometheus/prometheus/promql/engine.go:692
# 0x1c73ccb github.com/prometheus/prometheus/promql/parser.inspector.Visit+0x5b /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:293
# 0x1c73a18 github.com/prometheus/prometheus/promql/parser.Walk+0x78 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:275
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1d0bf09 github.com/prometheus/prometheus/promql/parser.Inspect+0x169 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:305
# 0x1d0be0f github.com/prometheus/prometheus/promql.(*Engine).populateSeries+0x6f /go/src/github.com/prometheus/prometheus/promql/engine.go:655
# 0x1d0a969 github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x319 /go/src/github.com/prometheus/prometheus/promql/engine.go:513
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8e849 github.com/prometheus/prometheus/web/api/v1.(*API).queryRange+0xb59 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:418
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
2 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1cdfa8d 0x1cdf8e7 0x1ce8aa0 0x1ce7c13 0x1cea81c 0x1ce62aa 0x1ce6250 0x1ce4d95 0x1d2629f 0x1c73ccc 0x1c73a19 0x1c73b63 0x1c73b63 0x1c73b63 0x1d0bf0a 0x1d0be10 0x1d0a96a 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1cdfa8c sync.(*RWMutex).RLock+0x48c /usr/local/go/src/sync/rwmutex.go:50
# 0x1cdf8e6 github.com/prometheus/prometheus/tsdb.(*headIndexReader).Postings+0x2e6 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1429
# 0x1ce8a9f github.com/prometheus/prometheus/tsdb.postingsForMatcher+0x4af /go/src/github.com/prometheus/prometheus/tsdb/querier.go:416
# 0x1ce7c12 github.com/prometheus/prometheus/tsdb.PostingsForMatchers+0x292 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:373
# 0x1cea81b github.com/prometheus/prometheus/tsdb.lookupChunkSeries+0x8b /go/src/github.com/prometheus/prometheus/tsdb/querier.go:711
# 0x1ce62a9 github.com/prometheus/prometheus/tsdb.LookupChunkSeries+0x89 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:698
# 0x1ce624f github.com/prometheus/prometheus/tsdb.(*blockQuerier).Select+0x2f /go/src/github.com/prometheus/prometheus/tsdb/querier.go:199
# 0x1ce4d94 github.com/prometheus/prometheus/tsdb.(*querier).Select+0x114 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:94
# 0x1d2629e github.com/prometheus/prometheus/promql.(*Engine).populateSeries.func1+0x3ce /go/src/github.com/prometheus/prometheus/promql/engine.go:692
# 0x1c73ccb github.com/prometheus/prometheus/promql/parser.inspector.Visit+0x5b /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:293
# 0x1c73a18 github.com/prometheus/prometheus/promql/parser.Walk+0x78 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:275
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1d0bf09 github.com/prometheus/prometheus/promql/parser.Inspect+0x169 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:305
# 0x1d0be0f github.com/prometheus/prometheus/promql.(*Engine).populateSeries+0x6f /go/src/github.com/prometheus/prometheus/promql/engine.go:655
# 0x1d0a969 github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x319 /go/src/github.com/prometheus/prometheus/promql/engine.go:513
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
2 @ 0x431340 0x4419b0 0x44199b 0x441717 0x46d57c 0x46ec47 0x46ebd2 0x1cdcbe1 0x1c6e158 0x1d852bf 0x1d81967 0x1d7ec3b 0x45f181
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x46d57b sync.(*Mutex).lockSlow+0xfb /usr/local/go/src/sync/mutex.go:138
# 0x46ec46 sync.(*Mutex).Lock+0x96 /usr/local/go/src/sync/mutex.go:81
# 0x46ebd1 sync.(*RWMutex).Lock+0x21 /usr/local/go/src/sync/rwmutex.go:98
# 0x1cdcbe0 github.com/prometheus/prometheus/tsdb.(*headAppender).AddFast+0x70 /go/src/github.com/prometheus/prometheus/tsdb/head.go:990
# 0x1c6e157 github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast+0x57 /go/src/github.com/prometheus/prometheus/storage/fanout.go:143
# 0x1d852be github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast+0x7e /go/src/github.com/prometheus/prometheus/scrape/target.go:339
# 0x1d81966 github.com/prometheus/prometheus/scrape.(*scrapeLoop).append+0x18a6 /go/src/github.com/prometheus/prometheus/scrape/scrape.go:1124
# 0x1d7ec3a github.com/prometheus/prometheus/scrape.(*scrapeLoop).run+0x5ea /go/src/github.com/prometheus/prometheus/scrape/scrape.go:966
2 @ 0x431340 0x4419b0 0x44199b 0x441717 0x46d57c 0x46ec47 0x46ebd2 0x1ce03bf 0x1ceabf7 0x1ceb5e4 0x1ceb7b8 0x1d0c45b 0x1d0c325 0x1d0c3c4 0x1d0f92a 0x1d0e218 0x1d0f1ea 0x1d0ca99 0x1d0ac2f 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x46d57b sync.(*Mutex).lockSlow+0xfb /usr/local/go/src/sync/mutex.go:138
# 0x46ec46 sync.(*Mutex).Lock+0x96 /usr/local/go/src/sync/mutex.go:81
# 0x46ebd1 sync.(*RWMutex).Lock+0x21 /usr/local/go/src/sync/rwmutex.go:98
# 0x1ce03be github.com/prometheus/prometheus/tsdb.(*headIndexReader).Series+0x11e /go/src/github.com/prometheus/prometheus/tsdb/head.go:1481
# 0x1ceabf6 github.com/prometheus/prometheus/tsdb.(*baseChunkSeries).Next+0x186 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:740
# 0x1ceb5e3 github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next+0x403 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:798
# 0x1ceb7b7 github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next+0x47 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:857
# 0x1d0c45a github.com/prometheus/prometheus/promql.expandSeriesSet+0x5a /go/src/github.com/prometheus/prometheus/promql/engine.go:756
# 0x1d0c324 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x94 /go/src/github.com/prometheus/prometheus/promql/engine.go:745
# 0x1d0c3c3 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x133 /go/src/github.com/prometheus/prometheus/promql/engine.go:742
# 0x1d0f929 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x10d9 /go/src/github.com/prometheus/prometheus/promql/engine.go:1097
# 0x1d0e217 github.com/prometheus/prometheus/promql.(*evaluator).rangeEval+0x1407 /go/src/github.com/prometheus/prometheus/promql/engine.go:887
# 0x1d0f1e9 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x999 /go/src/github.com/prometheus/prometheus/promql/engine.go:1032
# 0x1d0ca98 github.com/prometheus/prometheus/promql.(*evaluator).Eval+0x98 /go/src/github.com/prometheus/prometheus/promql/engine.go:814
# 0x1d0ac2e github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x5de /go/src/github.com/prometheus/prometheus/promql/engine.go:535
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
1 @ 0x40afb4 0x4459ec 0x789052 0x45f181
# 0x4459eb os/signal.signal_recv+0x9b /usr/local/go/src/runtime/sigqueue.go:147
# 0x789051 os/signal.loop+0x21 /usr/local/go/src/os/signal/signal_unix.go:23
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1b16354 0x1b2660c 0x45f181
# 0x1b16353 github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run+0x3a3 /go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:149
# 0x1b2660b github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15+0x8b /go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:459
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1b1e3fd 0x1b2660c 0x45f181
# 0x1b1e3fc github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run+0x30c /go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:100
# 0x1b2660b github.com/prometheus/prometheus/discovery/kubernetes.(*Discovery).Run.func15+0x8b /go/src/github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:459
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1e4b3bc 0x45f181
# 0x1e4b3bb github.com/prometheus/prometheus/prompb.RegisterAdminHandlerFromEndpoint.func1.1+0x4b /go/src/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go:202
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1ed70e8 0x867247 0x45f181
# 0x1ed70e7 main.main.func16+0x397 /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:636
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1ed71c2 0x867247 0x45f181
# 0x1ed71c1 main.main.func18+0x71 /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:652
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x1ed7bc9 0x867247 0x45f181
# 0x1ed7bc8 main.main.func20+0x988 /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:699
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0x86711d 0x1ecf840 0x430f6e 0x45f181
# 0x86711c github.com/oklog/run.(*Group).Run+0xec /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:43
# 0x1ecf83f main.main+0x6ebf /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:746
# 0x430f6d runtime.main+0x21d /usr/local/go/src/runtime/proc.go:203
1 @ 0x431340 0x405b78 0x405b4e 0x40583b 0xf2650a 0x45f181
# 0xf26509 google.golang.org/grpc.(*addrConn).resetTransport+0x6e9 /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/clientconn.go:1179
1 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x1b81612 0x1ed5ebe 0x867247 0x45f181
# 0x1b81611 github.com/prometheus/prometheus/discovery.(*Manager).Run+0x71 /go/src/github.com/prometheus/prometheus/discovery/manager.go:171
# 0x1ed5ebd main.main.func8+0x4d /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:530
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x1b81612 0x1ed617e 0x867247 0x45f181
# 0x1b81611 github.com/prometheus/prometheus/discovery.(*Manager).Run+0x71 /go/src/github.com/prometheus/prometheus/discovery/manager.go:171
# 0x1ed617d main.main.func10+0x4d /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:544
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x1e60fc7 0x45f181
# 0x1e60fc6 github.com/prometheus/prometheus/storage/remote.(*WriteStorage).run+0xc6 /go/src/github.com/prometheus/prometheus/storage/remote/write.go:83
1 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x1e8819b 0x6ef2f0 0x1ec69ff 0x45f181
# 0x1e8819a github.com/soheilhy/cmux.muxListener.Accept+0x5a /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:229
# 0x6ef2ef net/http.(*Server).Serve+0x27f /usr/local/go/src/net/http/server.go:2896
# 0x1ec69fe github.com/prometheus/prometheus/web.(*Handler).Run.func3+0x3e /go/src/github.com/prometheus/prometheus/web/web.go:582
1 @ 0x431340 0x405b78 0x405b4e 0x40587b 0x1e8819b 0xf31c6e 0x1ec6a7f 0x45f181
# 0x1e8819a github.com/soheilhy/cmux.muxListener.Accept+0x5a /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:229
# 0xf31c6d google.golang.org/grpc.(*Server).Serve+0x22d /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/server.go:597
# 0x1ec6a7e github.com/prometheus/prometheus/web.(*Handler).Run.func4+0x3e /go/src/github.com/prometheus/prometheus/web/web.go:585
1 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x1e84f5c 0x4cbee7 0xec9ab7 0xec9a6a 0xeca321 0x1e86708 0x1e87b9a 0x1e85d3e 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x1e84f5b github.com/soheilhy/cmux.(*bufferedReader).Read+0x12b /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/buffer.go:53
# 0x4cbee6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:310
# 0xec9ab6 io.ReadFull+0x86 /usr/local/go/src/io/io.go:329
# 0xec9a69 golang.org/x/net/http2.readFrameHeader+0x39 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237
# 0xeca320 golang.org/x/net/http2.(*Framer).ReadFrame+0xa0 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492
# 0x1e86707 github.com/soheilhy/cmux.matchHTTP2Field+0x147 /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/matchers.go:236
# 0x1e87b99 github.com/soheilhy/cmux.HTTP2MatchHeaderFieldSendSettings.func1+0xc9 /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/matchers.go:173
# 0x1e85d3d github.com/soheilhy/cmux.(*cMux).serve+0x1ed /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:184
1 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x5ba2fa 0x4cbee7 0xec9ab7 0xec9a6a 0xeca321 0xefae93 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x5ba2f9 bufio.(*Reader).Read+0x269 /usr/local/go/src/bufio/bufio.go:226
# 0x4cbee6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:310
# 0xec9ab6 io.ReadFull+0x86 /usr/local/go/src/io/io.go:329
# 0xec9a69 golang.org/x/net/http2.readFrameHeader+0x39 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237
# 0xeca320 golang.org/x/net/http2.(*Framer).ReadFrame+0xa0 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492
# 0xefae92 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x182 /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1265
1 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x61d1a0 0x5479d4 0x61d3ec 0x61b944 0x61fbb1 0x61fbbc 0x5ba2fa 0x4cbee7 0xec9ab7 0xec9a6a 0xeca321 0xedb4dc 0xedac33 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x61d19f crypto/tls.(*atLeastReader).Read+0x5f /usr/local/go/src/crypto/tls/conn.go:780
# 0x5479d3 bytes.(*Buffer).ReadFrom+0xb3 /usr/local/go/src/bytes/buffer.go:204
# 0x61d3eb crypto/tls.(*Conn).readFromUntil+0xeb /usr/local/go/src/crypto/tls/conn.go:802
# 0x61b943 crypto/tls.(*Conn).readRecordOrCCS+0x123 /usr/local/go/src/crypto/tls/conn.go:609
# 0x61fbb0 crypto/tls.(*Conn).readRecord+0x160 /usr/local/go/src/crypto/tls/conn.go:577
# 0x61fbbb crypto/tls.(*Conn).Read+0x16b /usr/local/go/src/crypto/tls/conn.go:1255
# 0x5ba2f9 bufio.(*Reader).Read+0x269 /usr/local/go/src/bufio/bufio.go:226
# 0x4cbee6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:310
# 0xec9ab6 io.ReadFull+0x86 /usr/local/go/src/io/io.go:329
# 0xec9a69 golang.org/x/net/http2.readFrameHeader+0x39 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:237
# 0xeca320 golang.org/x/net/http2.(*Framer).ReadFrame+0xa0 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/frame.go:492
# 0xedb4db golang.org/x/net/http2.(*clientConnReadLoop).run+0x8b /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1746
# 0xedac32 golang.org/x/net/http2.(*ClientConn).readLoop+0xa2 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/http2/transport.go:1674
1 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d0a7f 0x4d0a61 0x519bdf 0x52dbc8 0x61d1a0 0x5479d4 0x61d3ec 0x61b944 0x61fbb1 0x61fbbc 0x705595 0x5b9c33 0x5b9d9f 0x706246 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d0a7e internal/poll.(*pollDesc).waitRead+0x1ce /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d0a60 internal/poll.(*FD).Read+0x1b0 /usr/local/go/src/internal/poll/fd_unix.go:169
# 0x519bde net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_unix.go:202
# 0x52dbc7 net.(*conn).Read+0x67 /usr/local/go/src/net/net.go:184
# 0x61d19f crypto/tls.(*atLeastReader).Read+0x5f /usr/local/go/src/crypto/tls/conn.go:780
# 0x5479d3 bytes.(*Buffer).ReadFrom+0xb3 /usr/local/go/src/bytes/buffer.go:204
# 0x61d3eb crypto/tls.(*Conn).readFromUntil+0xeb /usr/local/go/src/crypto/tls/conn.go:802
# 0x61b943 crypto/tls.(*Conn).readRecordOrCCS+0x123 /usr/local/go/src/crypto/tls/conn.go:609
# 0x61fbb0 crypto/tls.(*Conn).readRecord+0x160 /usr/local/go/src/crypto/tls/conn.go:577
# 0x61fbbb crypto/tls.(*Conn).Read+0x16b /usr/local/go/src/crypto/tls/conn.go:1255
# 0x705594 net/http.(*persistConn).Read+0x74 /usr/local/go/src/net/http/transport.go:1758
# 0x5b9c32 bufio.(*Reader).fill+0x102 /usr/local/go/src/bufio/bufio.go:100
# 0x5b9d9e bufio.(*Reader).Peek+0x4e /usr/local/go/src/bufio/bufio.go:138
# 0x706245 net/http.(*persistConn).readLoop+0x1d5 /usr/local/go/src/net/http/transport.go:1911
1 @ 0x431340 0x42c0da 0x42b6a5 0x4cfb05 0x4d2488 0x4d2467 0x51a552 0x5386c2 0x537247 0x1e88be1 0x86470f 0x1e85a2f 0x1ec6af1 0x45f181
# 0x42b6a4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:184
# 0x4cfb04 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4d2487 internal/poll.(*pollDesc).waitRead+0x1f7 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4d2466 internal/poll.(*FD).Accept+0x1d6 /usr/local/go/src/internal/poll/fd_unix.go:384
# 0x51a551 net.(*netFD).accept+0x41 /usr/local/go/src/net/fd_unix.go:238
# 0x5386c1 net.(*TCPListener).accept+0x31 /usr/local/go/src/net/tcpsock_posix.go:139
# 0x537246 net.(*TCPListener).Accept+0x46 /usr/local/go/src/net/tcpsock.go:261
# 0x1e88be0 golang.org/x/net/netutil.(*limitListener).Accept+0x50 /go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil/listen.go:48
# 0x86470e github.com/mwitkow/go-conntrack.(*connTrackListener).Accept+0x7e /go/src/github.com/prometheus/prometheus/vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go:100
# 0x1e85a2e github.com/soheilhy/cmux.(*cMux).Serve+0xae /go/src/github.com/prometheus/prometheus/vendor/github.com/soheilhy/cmux/cmux.go:162
# 0x1ec6af0 github.com/prometheus/prometheus/web.(*Handler).Run.func5+0x30 /go/src/github.com/prometheus/prometheus/web/web.go:588
1 @ 0x431340 0x440e4b 0x1c530bd 0x1ed80cb 0x867247 0x45f181
# 0x1c530bc github.com/prometheus/prometheus/notifier.(*Manager).Run+0xfc /go/src/github.com/prometheus/prometheus/notifier/notifier.go:308
# 0x1ed80ca main.main.func24+0x8a /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:737
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x440e4b 0x1c5a3b3 0x45f181
# 0x1c5a3b2 github.com/prometheus/prometheus/pkg/logging.(*Deduper).run+0x1f2 /go/src/github.com/prometheus/prometheus/pkg/logging/dedupe.go:75
1 @ 0x431340 0x440e4b 0x1cb2f8b 0x45f181
# 0x1cb2f8a github.com/prometheus/prometheus/tsdb/wal.(*WAL).run+0xda /go/src/github.com/prometheus/prometheus/tsdb/wal/wal.go:321
1 @ 0x431340 0x440e4b 0x1ccddd3 0x45f181
# 0x1ccddd2 github.com/prometheus/prometheus/tsdb.(*DB).run+0x202 /go/src/github.com/prometheus/prometheus/tsdb/db.go:647
1 @ 0x431340 0x440e4b 0x1d7684c 0x1ed647b 0x867247 0x45f181
# 0x1d7684b github.com/prometheus/prometheus/scrape.(*Manager).Run+0xdb /go/src/github.com/prometheus/prometheus/scrape/manager.go:142
# 0x1ed647a main.main.func12+0x8a /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:564
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x440e4b 0x1d76a8a 0x45f181
# 0x1d76a89 github.com/prometheus/prometheus/scrape.(*Manager).reloader+0x1a9 /go/src/github.com/prometheus/prometheus/scrape/manager.go:166
1 @ 0x431340 0x440e4b 0x1ec0dc9 0x1ed7f80 0x867247 0x45f181
# 0x1ec0dc8 github.com/prometheus/prometheus/web.(*Handler).Run+0x1038 /go/src/github.com/prometheus/prometheus/web/web.go:591
# 0x1ed7f7f main.main.func22+0x3f /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:714
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x440e4b 0x1ed5bca 0x867247 0x45f181
# 0x1ed5bc9 main.main.func6+0xf9 /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:510
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x440e4b 0x1ed686d 0x867247 0x45f181
# 0x1ed686c main.main.func14+0x16c /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:589
# 0x867246 github.com/oklog/run.(*Group).Run.func1+0x26 /go/src/github.com/prometheus/prometheus/vendor/github.com/oklog/run/group.go:38
1 @ 0x431340 0x440e4b 0xeea7f2 0xeeb113 0xf0d09b 0x45f181
# 0xeea7f1 google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x121 /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395
# 0xeeb112 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1e2 /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513
# 0xf0d09a google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x7a /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346
1 @ 0x431340 0x440e4b 0xf1e302 0x45f181
# 0xf1e301 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0xc1 /go/src/github.com/prometheus/prometheus/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69
1 @ 0x431340 0x440e4b 0xf5dd70 0x45f181
# 0xf5dd6f go.opencensus.io/stats/view.(*worker).start+0xff /go/src/github.com/prometheus/prometheus/vendor/go.opencensus.io/stats/view/worker.go:154
1 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1cdfa8d 0x1cdf8e7 0x1ce89d7 0x1ce7c13 0x1cea81c 0x1ce62aa 0x1ce6250 0x1ce4d95 0x1d2629f 0x1c73ccc 0x1c73a19 0x1c73b63 0x1c73b63 0x1d0bf0a 0x1d0be10 0x1d0a96a 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1cdfa8c sync.(*RWMutex).RLock+0x48c /usr/local/go/src/sync/rwmutex.go:50
# 0x1cdf8e6 github.com/prometheus/prometheus/tsdb.(*headIndexReader).Postings+0x2e6 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1429
# 0x1ce89d6 github.com/prometheus/prometheus/tsdb.postingsForMatcher+0x3e6 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:424
# 0x1ce7c12 github.com/prometheus/prometheus/tsdb.PostingsForMatchers+0x292 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:373
# 0x1cea81b github.com/prometheus/prometheus/tsdb.lookupChunkSeries+0x8b /go/src/github.com/prometheus/prometheus/tsdb/querier.go:711
# 0x1ce62a9 github.com/prometheus/prometheus/tsdb.LookupChunkSeries+0x89 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:698
# 0x1ce624f github.com/prometheus/prometheus/tsdb.(*blockQuerier).Select+0x2f /go/src/github.com/prometheus/prometheus/tsdb/querier.go:199
# 0x1ce4d94 github.com/prometheus/prometheus/tsdb.(*querier).Select+0x114 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:94
# 0x1d2629e github.com/prometheus/prometheus/promql.(*Engine).populateSeries.func1+0x3ce /go/src/github.com/prometheus/prometheus/promql/engine.go:692
# 0x1c73ccb github.com/prometheus/prometheus/promql/parser.inspector.Visit+0x5b /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:293
# 0x1c73a18 github.com/prometheus/prometheus/promql/parser.Walk+0x78 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:275
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1c73b62 github.com/prometheus/prometheus/promql/parser.Walk+0x1c2 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:281
# 0x1d0bf09 github.com/prometheus/prometheus/promql/parser.Inspect+0x169 /go/src/github.com/prometheus/prometheus/promql/parser/ast.go:305
# 0x1d0be0f github.com/prometheus/prometheus/promql.(*Engine).populateSeries+0x6f /go/src/github.com/prometheus/prometheus/promql/engine.go:655
# 0x1d0a969 github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x319 /go/src/github.com/prometheus/prometheus/promql/engine.go:513
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
1 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1ce2339 0x1ce227b 0x1cdea69 0x1ceb312 0x1ceb7b8 0x1d0c45b 0x1d0c325 0x1d0c3c4 0x1d0f92a 0x1d0ca99 0x1d0ac2f 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea 0x1ec7916 0x1e845a6 0x1e7fc6e 0x1e8415c
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1ce2338 sync.(*RWMutex).RLock+0xe8 /usr/local/go/src/sync/rwmutex.go:50
# 0x1ce227a github.com/prometheus/prometheus/tsdb.(*memSeries).chunk+0x2a /go/src/github.com/prometheus/prometheus/tsdb/head.go:1832
# 0x1cdea68 github.com/prometheus/prometheus/tsdb.(*headChunkReader).Chunk+0x88 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1325
# 0x1ceb311 github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next+0x131 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:819
# 0x1ceb7b7 github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next+0x47 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:857
# 0x1d0c45a github.com/prometheus/prometheus/promql.expandSeriesSet+0x5a /go/src/github.com/prometheus/prometheus/promql/engine.go:756
# 0x1d0c324 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x94 /go/src/github.com/prometheus/prometheus/promql/engine.go:745
# 0x1d0c3c3 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x133 /go/src/github.com/prometheus/prometheus/promql/engine.go:742
# 0x1d0f929 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x10d9 /go/src/github.com/prometheus/prometheus/promql/engine.go:1097
# 0x1d0ca98 github.com/prometheus/prometheus/promql.(*evaluator).Eval+0x98 /go/src/github.com/prometheus/prometheus/promql/engine.go:814
# 0x1d0ac2e github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x5de /go/src/github.com/prometheus/prometheus/promql/engine.go:535
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
# 0x1ec7915 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1+0x1c5 /go/src/github.com/prometheus/prometheus/web/web.go:1122
# 0x1e845a5 github.com/prometheus/common/route.(*Router).handle.func1+0x295 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83
# 0x1e7fc6d github.com/julienschmidt/httprouter.(*Router).ServeHTTP+0x95d /go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:334
# 0x1e8415b github.com/prometheus/common/route.(*Router).ServeHTTP+0x4b /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:121
1 @ 0x431340 0x4419b0 0x44199b 0x441717 0x1ce2339 0x1ce227b 0x1cdea69 0x1ceb312 0x1ceb7b8 0x1d0c45b 0x1d0c325 0x1d0c3c4 0x1d0f92a 0x1d0e218 0x1d0f1ea 0x1d0e218 0x1d10eee 0x1d0ca99 0x1d0ac2f 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x1ce2338 sync.(*RWMutex).RLock+0xe8 /usr/local/go/src/sync/rwmutex.go:50
# 0x1ce227a github.com/prometheus/prometheus/tsdb.(*memSeries).chunk+0x2a /go/src/github.com/prometheus/prometheus/tsdb/head.go:1832
# 0x1cdea68 github.com/prometheus/prometheus/tsdb.(*headChunkReader).Chunk+0x88 /go/src/github.com/prometheus/prometheus/tsdb/head.go:1325
# 0x1ceb311 github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next+0x131 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:819
# 0x1ceb7b7 github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next+0x47 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:857
# 0x1d0c45a github.com/prometheus/prometheus/promql.expandSeriesSet+0x5a /go/src/github.com/prometheus/prometheus/promql/engine.go:756
# 0x1d0c324 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x94 /go/src/github.com/prometheus/prometheus/promql/engine.go:745
# 0x1d0c3c3 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x133 /go/src/github.com/prometheus/prometheus/promql/engine.go:742
# 0x1d0f929 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x10d9 /go/src/github.com/prometheus/prometheus/promql/engine.go:1097
# 0x1d0e217 github.com/prometheus/prometheus/promql.(*evaluator).rangeEval+0x1407 /go/src/github.com/prometheus/prometheus/promql/engine.go:887
# 0x1d0f1e9 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x999 /go/src/github.com/prometheus/prometheus/promql/engine.go:1032
# 0x1d0e217 github.com/prometheus/prometheus/promql.(*evaluator).rangeEval+0x1407 /go/src/github.com/prometheus/prometheus/promql/engine.go:887
# 0x1d10eed github.com/prometheus/prometheus/promql.(*evaluator).eval+0x269d /go/src/github.com/prometheus/prometheus/promql/engine.go:1077
# 0x1d0ca98 github.com/prometheus/prometheus/promql.(*evaluator).Eval+0x98 /go/src/github.com/prometheus/prometheus/promql/engine.go:814
# 0x1d0ac2e github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x5de /go/src/github.com/prometheus/prometheus/promql/engine.go:535
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
1 @ 0x431340 0x4419b0 0x44199b 0x441717 0x46d57c 0x46ec47 0x46ebd2 0x1ce03bf 0x1ceabf7 0x1ceb5e4 0x1ceb7b8 0x1d0c45b 0x1d0c325 0x1d0c3c4 0x1d0f92a 0x1d0ca99 0x1d0ac2f 0x1d0a106 0x1d07cf4 0x1e8d5e6 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea 0x1ec7916 0x1e845a6
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x46d57b sync.(*Mutex).lockSlow+0xfb /usr/local/go/src/sync/mutex.go:138
# 0x46ec46 sync.(*Mutex).Lock+0x96 /usr/local/go/src/sync/mutex.go:81
# 0x46ebd1 sync.(*RWMutex).Lock+0x21 /usr/local/go/src/sync/rwmutex.go:98
# 0x1ce03be github.com/prometheus/prometheus/tsdb.(*headIndexReader).Series+0x11e /go/src/github.com/prometheus/prometheus/tsdb/head.go:1481
# 0x1ceabf6 github.com/prometheus/prometheus/tsdb.(*baseChunkSeries).Next+0x186 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:740
# 0x1ceb5e3 github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next+0x403 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:798
# 0x1ceb7b7 github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next+0x47 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:857
# 0x1d0c45a github.com/prometheus/prometheus/promql.expandSeriesSet+0x5a /go/src/github.com/prometheus/prometheus/promql/engine.go:756
# 0x1d0c324 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x94 /go/src/github.com/prometheus/prometheus/promql/engine.go:745
# 0x1d0c3c3 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x133 /go/src/github.com/prometheus/prometheus/promql/engine.go:742
# 0x1d0f929 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x10d9 /go/src/github.com/prometheus/prometheus/promql/engine.go:1097
# 0x1d0ca98 github.com/prometheus/prometheus/promql.(*evaluator).Eval+0x98 /go/src/github.com/prometheus/prometheus/promql/engine.go:814
# 0x1d0ac2e github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0x5de /go/src/github.com/prometheus/prometheus/promql/engine.go:535
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8d5e5 github.com/prometheus/prometheus/web/api/v1.(*API).query+0x3b5 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:346
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
# 0x1ec7915 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1+0x1c5 /go/src/github.com/prometheus/prometheus/web/web.go:1122
# 0x1e845a5 github.com/prometheus/common/route.(*Router).handle.func1+0x295 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83
1 @ 0x431340 0x4419b0 0x44199b 0x441717 0x46d57c 0x46ec47 0x46ebd2 0x1ce03bf 0x1ceabf7 0x1ceb5e4 0x1ceb7b8 0x1d0c45b 0x1d0c325 0x1d0c3c4 0x1d0f92a 0x1d0e218 0x1d0f1ea 0x1d0ca99 0x1d0b5be 0x1d0a106 0x1d07cf4 0x1e8e84a 0x1e9c1ee 0x6eb9e4 0x1e8b02c 0x1ec6785 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea
# 0x441716 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71
# 0x46d57b sync.(*Mutex).lockSlow+0xfb /usr/local/go/src/sync/mutex.go:138
# 0x46ec46 sync.(*Mutex).Lock+0x96 /usr/local/go/src/sync/mutex.go:81
# 0x46ebd1 sync.(*RWMutex).Lock+0x21 /usr/local/go/src/sync/rwmutex.go:98
# 0x1ce03be github.com/prometheus/prometheus/tsdb.(*headIndexReader).Series+0x11e /go/src/github.com/prometheus/prometheus/tsdb/head.go:1481
# 0x1ceabf6 github.com/prometheus/prometheus/tsdb.(*baseChunkSeries).Next+0x186 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:740
# 0x1ceb5e3 github.com/prometheus/prometheus/tsdb.(*populatedChunkSeries).Next+0x403 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:798
# 0x1ceb7b7 github.com/prometheus/prometheus/tsdb.(*blockSeriesSet).Next+0x47 /go/src/github.com/prometheus/prometheus/tsdb/querier.go:857
# 0x1d0c45a github.com/prometheus/prometheus/promql.expandSeriesSet+0x5a /go/src/github.com/prometheus/prometheus/promql/engine.go:756
# 0x1d0c324 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x94 /go/src/github.com/prometheus/prometheus/promql/engine.go:745
# 0x1d0c3c3 github.com/prometheus/prometheus/promql.checkForSeriesSetExpansion+0x133 /go/src/github.com/prometheus/prometheus/promql/engine.go:742
# 0x1d0f929 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x10d9 /go/src/github.com/prometheus/prometheus/promql/engine.go:1097
# 0x1d0e217 github.com/prometheus/prometheus/promql.(*evaluator).rangeEval+0x1407 /go/src/github.com/prometheus/prometheus/promql/engine.go:887
# 0x1d0f1e9 github.com/prometheus/prometheus/promql.(*evaluator).eval+0x999 /go/src/github.com/prometheus/prometheus/promql/engine.go:1032
# 0x1d0ca98 github.com/prometheus/prometheus/promql.(*evaluator).Eval+0x98 /go/src/github.com/prometheus/prometheus/promql/engine.go:814
# 0x1d0b5bd github.com/prometheus/prometheus/promql.(*Engine).execEvalStmt+0xf6d /go/src/github.com/prometheus/prometheus/promql/engine.go:584
# 0x1d0a105 github.com/prometheus/prometheus/promql.(*Engine).exec+0x4c5 /go/src/github.com/prometheus/prometheus/promql/engine.go:486
# 0x1d07cf3 github.com/prometheus/prometheus/promql.(*query).Exec+0x93 /go/src/github.com/prometheus/prometheus/promql/engine.go:188
# 0x1e8e849 github.com/prometheus/prometheus/web/api/v1.(*API).queryRange+0xb59 /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:418
# 0x1e9c1ed github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1+0x8d /go/src/github.com/prometheus/prometheus/web/api/v1/api.go:251
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e8b02b github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP+0x7b /go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90
# 0x1ec6784 github.com/prometheus/prometheus/web.(*Handler).testReady.func1+0x54 /go/src/github.com/prometheus/prometheus/web/web.go:489
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
1 @ 0x431340 0x4429b8 0x44298e 0x46b72d 0x122c950 0x1b163df 0x1b245c9 0x45f181
# 0x44298d sync.runtime_notifyListWait+0xcd /usr/local/go/src/runtime/sema.go:510
# 0x46b72c sync.(*Cond).Wait+0x9c /usr/local/go/src/sync/cond.go:56
# 0x122c94f k8s.io/client-go/util/workqueue.(*Type).Get+0x9f /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/util/workqueue/queue.go:145
# 0x1b163de github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).process+0x4e /go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:153
# 0x1b245c8 github.com/prometheus/prometheus/discovery/kubernetes.(*Endpoints).Run.func1+0x48 /go/src/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go:144
1 @ 0x431340 0x4429b8 0x44298e 0x46b72d 0x122c950 0x1b1e48f 0x1b26809 0x45f181
# 0x44298d sync.runtime_notifyListWait+0xcd /usr/local/go/src/runtime/sema.go:510
# 0x46b72c sync.(*Cond).Wait+0x9c /usr/local/go/src/sync/cond.go:56
# 0x122c94f k8s.io/client-go/util/workqueue.(*Type).Get+0x9f /go/src/github.com/prometheus/prometheus/vendor/k8s.io/client-go/util/workqueue/queue.go:145
# 0x1b1e48e github.com/prometheus/prometheus/discovery/kubernetes.(*Node).process+0x4e /go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:104
# 0x1b26808 github.com/prometheus/prometheus/discovery/kubernetes.(*Node).Run.func1+0x48 /go/src/github.com/prometheus/prometheus/discovery/kubernetes/node.go:95
1 @ 0x46ed32 0x1cdcc26 0x1c6e158 0x1d852bf 0x1d81967 0x1d7ec3b 0x45f181
# 0x46ed31 sync.(*RWMutex).Unlock+0xd1 /usr/local/go/src/sync/rwmutex.go:118
# 0x1cdcc25 github.com/prometheus/prometheus/tsdb.(*headAppender).AddFast+0xb5 /go/src/github.com/prometheus/prometheus/tsdb/head.go:996
# 0x1c6e157 github.com/prometheus/prometheus/storage.(*fanoutAppender).AddFast+0x57 /go/src/github.com/prometheus/prometheus/storage/fanout.go:143
# 0x1d852be github.com/prometheus/prometheus/scrape.(*timeLimitAppender).AddFast+0x7e /go/src/github.com/prometheus/prometheus/scrape/target.go:339
# 0x1d81966 github.com/prometheus/prometheus/scrape.(*scrapeLoop).append+0x18a6 /go/src/github.com/prometheus/prometheus/scrape/scrape.go:1124
# 0x1d7ec3a github.com/prometheus/prometheus/scrape.(*scrapeLoop).run+0x5ea /go/src/github.com/prometheus/prometheus/scrape/scrape.go:966
1 @ 0x77e315 0x77e130 0x77ad5a 0x78786a 0x788281 0x1ebf8af 0x6eb9e4 0x1e6f749 0x6eb9e4 0x1e6f302 0x6eb9e4 0x1e6f4ea 0x1ec7916 0x1e845a6 0x1e7fc6e 0x1e8415c 0x6ed9bd 0x1e68e1b 0x6eb9e4 0x1ec5613 0x6eb9e4 0x6eef34 0x6ea7d5 0x45f181
# 0x77e314 runtime/pprof.writeRuntimeProfile+0x94 /usr/local/go/src/runtime/pprof/pprof.go:708
# 0x77e12f runtime/pprof.writeGoroutine+0x9f /usr/local/go/src/runtime/pprof/pprof.go:670
# 0x77ad59 runtime/pprof.(*Profile).WriteTo+0x3d9 /usr/local/go/src/runtime/pprof/pprof.go:329
# 0x787869 net/http/pprof.handler.ServeHTTP+0x339 /usr/local/go/src/net/http/pprof/pprof.go:245
# 0x788280 net/http/pprof.Index+0x6f0 /usr/local/go/src/net/http/pprof/pprof.go:268
# 0x1ebf8ae github.com/prometheus/prometheus/web.serveDebug+0x1ce /go/src/github.com/prometheus/prometheus/web/web.go:470
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f748 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1+0xe8 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:196
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f301 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2+0xb1 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:76
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1e6f4e9 github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1+0xd9 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go:100
# 0x1ec7915 github.com/prometheus/prometheus/web.setPathWithPrefix.func1.1+0x1c5 /go/src/github.com/prometheus/prometheus/web/web.go:1122
# 0x1e845a5 github.com/prometheus/common/route.(*Router).handle.func1+0x295 /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83
# 0x1e7fc6d github.com/julienschmidt/httprouter.(*Router).ServeHTTP+0x95d /go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:334
# 0x1e8415b github.com/prometheus/common/route.(*Router).ServeHTTP+0x4b /go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:121
# 0x6ed9bc net/http.(*ServeMux).ServeHTTP+0x1bc /usr/local/go/src/net/http/server.go:2387
# 0x1e68e1a github.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5+0x5fa /go/src/github.com/prometheus/prometheus/vendor/github.com/opentracing-contrib/go-stdlib/nethttp/server.go:140
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x1ec5612 github.com/prometheus/prometheus/web.withStackTracer.func1+0xb2 /go/src/github.com/prometheus/prometheus/web/web.go:103
# 0x6eb9e3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2007
# 0x6eef33 net/http.serverHandler.ServeHTTP+0xa3 /usr/local/go/src/net/http/server.go:2802
# 0x6ea7d4 net/http.(*conn).serve+0x874 /usr/local/go/src/net/http/server.go:1890
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment