Skip to content

Instantly share code, notes, and snippets.

@TimSimmons
Last active June 15, 2017 14:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save TimSimmons/56841047eecdf8011fc29dddff99da67 to your computer and use it in GitHub Desktop.
Save TimSimmons/56841047eecdf8011fc29dddff99da67 to your computer and use it in GitHub Desktop.
Prometheus hanging during a checkpoint Goroutine dump
goroutine 184516782 [running]:
runtime/pprof.writeGoroutineStacks(0x274c720, 0xc6667650a0, 0x30, 0xc8123126c0)
/usr/local/go/src/runtime/pprof/pprof.go:603 +0x79
runtime/pprof.writeGoroutine(0x274c720, 0xc6667650a0, 0x2, 0xc6b1b0a870, 0x40fe28)
/usr/local/go/src/runtime/pprof/pprof.go:592 +0x44
runtime/pprof.(*Profile).WriteTo(0x27fa5c0, 0x274c720, 0xc6667650a0, 0x2, 0xc6667650a0, 0xc6b1b0aaa0)
/usr/local/go/src/runtime/pprof/pprof.go:302 +0x3b5
net/http/pprof.handler.ServeHTTP(0xc812312461, 0x9, 0x275b560, 0xc6667650a0, 0xc826701100)
/usr/local/go/src/net/http/pprof/pprof.go:209 +0x1d1
net/http/pprof.Index(0x275b560, 0xc6667650a0, 0xc826701100)
/usr/local/go/src/net/http/pprof/pprof.go:221 +0x1e3
net/http.HandlerFunc.ServeHTTP(0x1c133e8, 0x275b560, 0xc6667650a0, 0xc826701100)
/usr/local/go/src/net/http/server.go:1942 +0x44
net/http.(*ServeMux).ServeHTTP(0x280b820, 0x275b560, 0xc6667650a0, 0xc826701100)
/usr/local/go/src/net/http/server.go:2238 +0x130
net/http.(*ServeMux).ServeHTTP-fm(0x275b560, 0xc6667650a0, 0xc826701100)
/go/src/github.com/prometheus/prometheus/web/web.go:214 +0x48
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x275b560, 0xc6667650a0, 0xc826701100, 0xc80daf55c0, 0x1, 0x1)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:84 +0x2a8
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc420396040, 0x275b560, 0xc6667650a0, 0xc826701100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x75d
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0xc420645940, 0x275b560, 0xc6667650a0, 0xc826701100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:126 +0x4c
net/http.serverHandler.ServeHTTP(0xc5df274000, 0x275b560, 0xc6667650a0, 0xc826701100)
/usr/local/go/src/net/http/server.go:2568 +0x92
net/http.(*conn).serve(0xc7e004e1e0, 0x275cb20, 0xc7dd620c00)
/usr/local/go/src/net/http/server.go:1825 +0x612
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2668 +0x2ce
goroutine 1 [select, 21873 minutes]:
main.Main(0x0)
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:231 +0x18ce
main.main()
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:45 +0x22
goroutine 49 [chan receive]:
github.com/prometheus/prometheus/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x280bfa0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:879 +0x7a
created by github.com/prometheus/prometheus/vendor/github.com/golang/glog.init.1
/go/src/github.com/prometheus/prometheus/vendor/github.com/golang/glog/glog.go:410 +0x21d
goroutine 243 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x45b8db619ce6f844)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*persistence).checkpointSeriesMapAndHeads.func3(0xc420644080, 0xc420ffcce0, 0xc420ffcc88, 0xc420ffcd58, 0xc5f6ef5f00, 0xc4202d0580)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:700 +0x50
github.com/prometheus/prometheus/storage/local.(*persistence).checkpointSeriesMapAndHeads(0xc4202d0580, 0x275cb20, 0xc47c46ff80, 0xc421008be0, 0xc420644080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:806 +0x4e5
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop.func2(0xc47c4d48c0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1457 +0x149
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop.func3(0x7fb0d66340d0, 0xc47c46ff80, 0xc5b4b723c0, 0xc47c46f580, 0xc47c4d48c0, 0xc59259bb60, 0xc5b4b72360, 0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1488 +0x283
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1494 +0x2ed
goroutine 100 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc42034c2a0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x23f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x19c
goroutine 39 [syscall, 21876 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 187 [select, 10731 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420108000)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5d9
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x583
goroutine 201 [select, 21873 minutes, locked to thread]:
runtime.gopark(0x1c13830, 0x0, 0x1b8cb44, 0x6, 0x18, 0x2)
/usr/local/go/src/runtime/proc.go:271 +0x13a
runtime.selectgoImpl(0xc420471f50, 0x0, 0x18)
/usr/local/go/src/runtime/select.go:423 +0x1364
runtime.selectgo(0xc420471f50)
/usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:434 +0x2dd
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2197 +0x1
goroutine 200 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0xc420316aa0)
/go/src/github.com/prometheus/prometheus/rules/manager.go:179 +0x3b7
github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1.1(0xc4203205d0, 0xc420316aa0)
/go/src/github.com/prometheus/prometheus/rules/manager.go:440 +0x52
created by github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1
/go/src/github.com/prometheus/prometheus/rules/manager.go:441 +0x56
goroutine 202 [select, 21873 minutes]:
main.Main.func2(0xc420268180, 0xc420268120, 0xc420071380, 0x6, 0x6, 0xc420278000)
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:172 +0x33f
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:186 +0xfd4
goroutine 203 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc420278620)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x23f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x19c
goroutine 188 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420108000)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5a5
goroutine 189 [select, 10731 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420108000)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x638
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x722
goroutine 190 [select, 10731 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420108000)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x242
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x744
goroutine 191 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc42034c0e0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x23f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x19c
goroutine 204 [select, 10868 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420108180)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5d9
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x583
goroutine 205 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420108180)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5a5
goroutine 206 [select, 10868 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420108180)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x638
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x722
goroutine 207 [select, 10870 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420108180)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x242
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x744
goroutine 101 [select, 11667 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420108300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5d9
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x583
goroutine 102 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420108300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5a5
goroutine 103 [select, 11667 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420108300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x638
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x722
goroutine 104 [select, 21876 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420108300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x242
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x744
goroutine 105 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc42034c540)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x23f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x19c
goroutine 106 [select, 11667 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc420108480)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x5d9
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x583
goroutine 107 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc420108480)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x249
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x5a5
goroutine 108 [select, 11667 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc420108480)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x638
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x722
goroutine 109 [select, 11667 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc420108480)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x242
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x744
goroutine 110 [select, 11666 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).OpenTransaction(0xc420108300, 0x0, 0x0, 0x453c20)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go:293 +0x42f
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).Write(0xc420108300, 0xc583dc6dc0, 0xc421006cc8, 0x1749320, 0xc5bbe8c410)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go:251 +0x612
github.com/prometheus/prometheus/storage/local/index.(*LevelDB).Commit(0xc4210088c0, 0x2754fe0, 0xc6aec0f580, 0x274d5e0, 0xc81f14b590)
/go/src/github.com/prometheus/prometheus/storage/local/index/leveldb.go:138 +0x62
github.com/prometheus/prometheus/storage/local/index.(*LabelPairFingerprintIndex).IndexBatch(0xc421006cf0, 0xc5d6ad9f80, 0xc4202d0580, 0xed0c8f206)
/go/src/github.com/prometheus/prometheus/storage/local/index/index.go:220 +0x1b4
github.com/prometheus/prometheus/storage/local.(*persistence).processIndexingQueue.func1()
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:1409 +0xe8
github.com/prometheus/prometheus/storage/local.(*persistence).processIndexingQueue(0xc4202d0580)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:1439 +0x2df
github.com/prometheus/prometheus/storage/local.(*persistence).run(0xc4202d0580)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:360 +0x2b
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:411 +0x129
goroutine 215 [select, 11667 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleQuarantine(0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1873 +0x41e
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:441 +0x47f
goroutine 214 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleEvictList(0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1107 +0x4be
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:440 +0x45a
goroutine 242 [select, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints.func1(0xc5b4b722a0, 0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1405 +0x382
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1420 +0x67
goroutine 241 [select, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints.func1(0xc5b4b72240, 0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1352 +0x43a
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1377 +0x67
goroutine 216 [select, 21873 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).logThrottling(0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:974 +0x6b8
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:442 +0x4a4
goroutine 217 [chan send, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*persistence).purgeArchivedMetric(0xc4202d0580, 0x355edf667d16bc2, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:1272 +0x2a5
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).maintainArchivedSeries(0xc4201dc000, 0x355edf667d16bc2, 0x15c73192e73)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1757 +0x272
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop(0xc4201dc000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1523 +0x576
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:443 +0x4c9
goroutine 221 [select, 21873 minutes]:
github.com/prometheus/prometheus/notifier.(*Notifier).Run(0xc42051a480)
/go/src/github.com/prometheus/prometheus/notifier/notifier.go:240 +0x1bf
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:211 +0x12f1
goroutine 223 [semacquire, 21873 minutes]:
sync.runtime_Semacquire(0xc420249db4)
/usr/local/go/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc420249da8)
/usr/local/go/src/sync/waitgroup.go:131 +0x7a
github.com/prometheus/prometheus/retrieval.(*TargetManager).Run(0xc420249d50)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:70 +0x159
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:217 +0x1395
goroutine 224 [IO wait]:
net.runtime_pollWait(0x7fb0d6633ec0, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc5ae8b6068, 0x72, 0x0, 0xc7a247ee40)
/usr/local/go/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc5ae8b6068, 0xffffffffffffffff, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).accept(0xc5ae8b6000, 0x0, 0x274ca20, 0xc7a247ee40)
/usr/local/go/src/net/fd_unix.go:430 +0x1e5
net.(*TCPListener).accept(0xc5d285c000, 0x403b2d, 0x183d960, 0xc420504000)
/usr/local/go/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc5d285c000, 0xc420504000, 0xc420a0de38, 0x4336fb, 0xc420a0de40)
/usr/local/go/src/net/tcpsock.go:228 +0x49
github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil.(*limitListener).Accept(0xc552b38040, 0x1c13208, 0xc7e004e1e0, 0x275cbe0, 0xc5da120180)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/netutil/listen.go:30 +0x5f
net/http.(*Server).Serve(0xc5df274000, 0x27561a0, 0xc552b38040, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2643 +0x228
github.com/prometheus/prometheus/web.(*Handler).Run(0xc420278000)
/go/src/github.com/prometheus/prometheus/web/web.go:271 +0x317
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:224 +0x1408
goroutine 257 [select, 5 minutes]:
github.com/prometheus/prometheus/discovery.(*TargetSet).Run(0xc59ce90820, 0x7fb0d66340d0, 0xc47ba56a40)
/go/src/github.com/prometheus/prometheus/discovery/discovery.go:192 +0x2e4
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x7fb0d66340d0, 0xc47ba56a40, 0xc420249d50, 0xc5c4c80180)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:114 +0x43
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:117 +0x605
goroutine 258 [select, 5 minutes]:
github.com/prometheus/prometheus/discovery/file.(*Discovery).Run(0xc5c4c801e0, 0x7fb0d66340d0, 0xc47b928ac0, 0xc420349f20)
/go/src/github.com/prometheus/prometheus/discovery/file/file.go:125 +0x430
created by github.com/prometheus/prometheus/discovery.(*TargetSet).updateProviders
/go/src/github.com/prometheus/prometheus/discovery/discovery.go:239 +0x283
goroutine 259 [select, 5 minutes]:
github.com/prometheus/prometheus/discovery.(*TargetSet).updateProviders.func1(0xc5c4c7e090, 0xc420349f20, 0xc59ce90820, 0xc5c4c7e0b0, 0xc5c4c7e080, 0x6, 0x2746920, 0xc5c4c801e0)
/go/src/github.com/prometheus/prometheus/discovery/discovery.go:262 +0x370
created by github.com/prometheus/prometheus/discovery.(*TargetSet).updateProviders
/go/src/github.com/prometheus/prometheus/discovery/discovery.go:276 +0x2f4
goroutine 227 [syscall, 3070 minutes]:
syscall.Syscall6(0xe8, 0x3b, 0xc5a0007cb4, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x2, 0x2, 0xc5a0017ec0)
/usr/local/go/src/syscall/asm_linux_amd64.s:44 +0x5
github.com/prometheus/prometheus/vendor/golang.org/x/sys/unix.EpollWait(0x3b, 0xc5a0007cb4, 0x7, 0x7, 0xffffffffffffffff, 0xc5e0646000, 0x0, 0xc5a120b940)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go:1220 +0x77
github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify%2ev1.(*fdPoller).wait(0xc551565e80, 0xc593b08600, 0x0, 0x1b88e01)
/go/src/github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify.v1/inotify_poller.go:86 +0x91
github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify%2ev1.(*Watcher).readEvents(0xc5b7e33ae0)
/go/src/github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify.v1/inotify.go:192 +0x196
created by github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify%2ev1.NewWatcher
/go/src/github.com/prometheus/prometheus/vendor/gopkg.in/fsnotify.v1/inotify.go:61 +0x2c8
goroutine 232 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xafaa93a9b51ada8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c0d3af20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c0d3af20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c0d3af20, 0x28006b0, 0xc77eaa6ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ce4edfe0, 0xc7c0d3af20, 0xc598db9a90, 0xc77eaa6f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ce4edfe0, 0xc5bfd50db0, 0xc7c0d3af20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4c15b4000, 0xc7c0d3af20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb476c0, 0xc821f50000, 0x951, 0xc00, 0x2e860896, 0x280bcc0, 0xc821f50000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb476c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 233 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf0dd08469a086b2b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc452c6a9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc452c6a9a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc452c6a9a0, 0x28006b0, 0xc7afa43440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc56805f940, 0xc452c6a9a0, 0xc602435a90, 0xc7afa434f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc56805f940, 0xc5bfd50e10, 0xc452c6a9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc56805f960, 0xc452c6a9a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47810, 0xc8829c8800, 0x8f0, 0x900, 0x6a0dac3, 0x280bcc0, 0xc8829c8800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47810, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 234 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x16c674d4ed21b473)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc648553ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc648553ee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc648553ee0, 0x28006b0, 0xc5bf6a18c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5543d9cc0, 0xc648553ee0, 0xc5d39b9a90, 0xc5bf6a1978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5543d9cc0, 0xc5bfd50e70, 0xc648553ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5543d9d00, 0xc648553ee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47960, 0xc8c57a1800, 0x7fe, 0x900, 0x2424d4a1, 0x280bcc0, 0xc8c57a1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47960, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 235 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3175b0195ae34d03)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc793a59ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc793a59ce0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc793a59ce0, 0x28006b0, 0xc7a47519e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e217f420, 0xc793a59ce0, 0xc5de191a90, 0xc7a4751a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e217f420, 0xc5bfd50ed0, 0xc793a59ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e217f440, 0xc793a59ce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47ab0, 0xc8b6f8b800, 0x827, 0x900, 0x22701636, 0x280bcc0, 0xc8b6f8b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47ab0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 236 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x57564d5a57be59de)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc68b603c80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc68b603c80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc68b603c80, 0x28006b0, 0xc81a7557a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83771dda0, 0xc68b603c80, 0xc5cc9cfa90, 0xc81a755858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83771dda0, 0xc5bfd50f30, 0xc68b603c80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83771ddc0, 0xc68b603c80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47c70, 0xc860394000, 0xa1e, 0xc00, 0x1dedd433, 0x280bcc0, 0xc860394000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47c70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 237 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2a44e270e29d1108)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc810edec00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc810edec00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc810edec00, 0x28006b0, 0xc68c222b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4e77d6fe0, 0xc810edec00, 0xc5cd875a90, 0xc68c222bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4e77d6fe0, 0xc5bfd50f90, 0xc810edec00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4e77d7060, 0xc810edec00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47dc0, 0xc8a8aba800, 0x854, 0x900, 0x243819ba, 0x280bcc0, 0xc8a8aba800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47dc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 238 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2fbe2ccd3d2e0006)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82de48780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82de48780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82de48780, 0x28006b0, 0xc69b4377a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7da542ec0, 0xc82de48780, 0xc5d9293a90, 0xc69b437858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7da542ec0, 0xc5bfd50ff0, 0xc82de48780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7da542ee0, 0xc82de48780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5abb47f10, 0xc8117be000, 0x393, 0x6a0, 0x2f8945ee, 0x280bcc0, 0xc8117be000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5abb47f10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 239 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf451374b741ebb34)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b22ab840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b22ab840, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b22ab840, 0x28006b0, 0xc71115ed80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f3fac420, 0xc7b22ab840, 0xc5dce5ba90, 0xc71115ee38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f3fac420, 0xc5bfd51050, 0xc7b22ab840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f3fac440, 0xc7b22ab840, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c070, 0xc8cb958800, 0x726, 0x900, 0x1848bfef, 0x280bcc0, 0xc8cb958800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 240 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2edafc121d3eb0c9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56dce9be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56dce9be0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56dce9be0, 0x28006b0, 0xc836f0e6c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc555cf2da0, 0xc56dce9be0, 0xc5e41b1a90, 0xc836f0e778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc555cf2da0, 0xc5bfd510b0, 0xc56dce9be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc555cf2dc0, 0xc56dce9be0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c1c0, 0xc982ae4000, 0x73e, 0x900, 0x3b7f2928, 0x280bcc0, 0xc982ae4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c1c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 273 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe6b75b96946d17dc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc75f1d3680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc75f1d3680, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc75f1d3680, 0x28006b0, 0xc75372c7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc609a04320, 0xc75f1d3680, 0xc598619a90, 0xc75372c898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc609a04320, 0xc5bfd51110, 0xc75f1d3680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc609a04340, 0xc75f1d3680, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c310, 0xc8cb3b1000, 0x835, 0x900, 0xc93053e, 0x280bcc0, 0xc8cb3b1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 274 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x138d7e8a34acdc09)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7316765c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7316765c0, 0x1a753c0, 0xc5cb6eba18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7316765c0, 0x28006b0, 0xc7670b9680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc662591120, 0xc7316765c0, 0xc5cb6eba90, 0xc7670b9738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc662591120, 0xc5bfd51170, 0xc7316765c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc662591140, 0xc7316765c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c460, 0xc8cb27a800, 0x6d1, 0x900, 0x1588a13b, 0x280bcc0, 0xc8cb27a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 275 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xabba03fb28aff373)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc87f05d580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc87f05d580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc87f05d580, 0x28006b0, 0xc88c7eaea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc775114940, 0xc87f05d580, 0xc5ddd2fa90, 0xc88c7eaf58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc775114940, 0xc5bfd511d0, 0xc87f05d580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7751149c0, 0xc87f05d580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c5b0, 0xc764f39c00, 0xa2, 0xc8, 0x36db29f0, 0x280bcc0, 0xc764f39c00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c5b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 276 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x98fa4adb679f2421)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc87356cbe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc87356cbe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc87356cbe0, 0x28006b0, 0xc7b00fafc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc621f13280, 0xc87356cbe0, 0xc63f1dda90, 0xc7b00fb078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc621f13280, 0xc5bfd51230, 0xc87356cbe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc621f132a0, 0xc87356cbe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c700, 0xc8b88d6800, 0x6ed, 0x900, 0x1b7df7f8, 0x280bcc0, 0xc8b88d6800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 277 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5ebb808810b00ffb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8725cd340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8725cd340, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8725cd340, 0x28006b0, 0xc7a008b0e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc833254ee0, 0xc8725cd340, 0xc598dbda90, 0xc7a008b198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc833254ee0, 0xc5bfd51290, 0xc8725cd340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc833254f00, 0xc8725cd340, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c850, 0xc7d61f8000, 0x798, 0x900, 0x8769ca4, 0x280bcc0, 0xc7d61f8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 278 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb27145b663ca5c0e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc551573d60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc551573d60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc551573d60, 0x28006b0, 0xc6d8b71320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7332b3460, 0xc551573d60, 0xc5d8891a90, 0xc6d8b713d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7332b3460, 0xc5bfd512f0, 0xc551573d60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7332b3480, 0xc551573d60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5c9a0, 0xc857bac800, 0x79b, 0x900, 0x2a51f339, 0x280bcc0, 0xc857bac800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5c9a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 279 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd0f18ed6478094d0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7df80fc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7df80fc40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7df80fc40, 0x28006b0, 0xc6229e8120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5df71bc20, 0xc7df80fc40, 0xc5aad8da90, 0xc6229e81d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5df71bc20, 0xc5bfd51350, 0xc7df80fc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5df71bc40, 0xc7df80fc40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5caf0, 0xc7f54cd000, 0x89d, 0x900, 0xcd06645, 0x280bcc0, 0xc7f54cd000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5caf0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 280 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe07ec428e6662fc8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc51086cd20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc51086cd20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc51086cd20, 0x28006b0, 0xc8569a5c20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8905b3a40, 0xc51086cd20, 0xc5e40e3a90, 0xc8569a5cd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8905b3a40, 0xc5bfd513b0, 0xc51086cd20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8905b3a80, 0xc51086cd20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5cc40, 0xc838fce000, 0x916, 0xc00, 0x23b5e41d, 0x280bcc0, 0xc838fce000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5cc40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 281 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfe9b31cb00b7f819)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5246aab40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5246aab40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5246aab40, 0x28006b0, 0xc81289f320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc56eb5ae60, 0xc5246aab40, 0xc586acda90, 0xc81289f3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc56eb5ae60, 0xc5bfd51410, 0xc5246aab40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc56eb5ae80, 0xc5246aab40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5cd90, 0xc87c1db500, 0x60a, 0x6a0, 0x1bbe0a81, 0x280bcc0, 0xc87c1db500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5cd90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 282 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe88f8ee8533da5f2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88aaa1720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88aaa1720, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88aaa1720, 0x28006b0, 0xc88f12cb40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e675caa0, 0xc88aaa1720, 0xc5ab2d9a90, 0xc88f12cbf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e675caa0, 0xc5bfd51470, 0xc88aaa1720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e675cae0, 0xc88aaa1720, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5cee0, 0xc8c5f3b800, 0x6e4, 0x900, 0x381d3e3e, 0x280bcc0, 0xc8c5f3b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5cee0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 283 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb540aec4b7d9d8f8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4df805e80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4df805e80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4df805e80, 0x28006b0, 0xc8371fd440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54da8a7a0, 0xc4df805e80, 0xc592599a90, 0xc8371fd4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54da8a7a0, 0xc5bfd514d0, 0xc4df805e80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54da8a7c0, 0xc4df805e80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d030, 0xc7b2ac2000, 0x910, 0xc00, 0x2ae03019, 0x280bcc0, 0xc7b2ac2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d030, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 284 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8464317ee3fb11bf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc625bf8c40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc625bf8c40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc625bf8c40, 0x28006b0, 0xc8ab519b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70050c5e0, 0xc625bf8c40, 0xc5e5e4da90, 0xc8ab519bc8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70050c5e0, 0xc5bfd51530, 0xc625bf8c40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70050c600, 0xc625bf8c40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d1f0, 0xc882b5ea00, 0x68b, 0x6a0, 0x26d1a1bd, 0x280bcc0, 0xc882b5ea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d1f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 285 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x19708a981cc24560)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8144ae8a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8144ae8a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8144ae8a0, 0x28006b0, 0xc78a07b7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6583aa6c0, 0xc8144ae8a0, 0xc5de963a90, 0xc78a07b858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6583aa6c0, 0xc5bfd51590, 0xc8144ae8a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6583aa720, 0xc8144ae8a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d340, 0xc8b17a3800, 0x8f5, 0x900, 0x13e36b06, 0x280bcc0, 0xc8b17a3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 286 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe786f4dcf02df977)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b45b7ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b45b7ec0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b45b7ec0, 0x28006b0, 0xc6a36f8120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc749282060, 0xc7b45b7ec0, 0xc605803a90, 0xc6a36f81d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc749282060, 0xc5bfd515f0, 0xc7b45b7ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc749282080, 0xc7b45b7ec0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d490, 0xc88e864000, 0x90b, 0xc00, 0x1f59a833, 0x280bcc0, 0xc88e864000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 287 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe9da797f41f94963)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc748056980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc748056980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc748056980, 0x28006b0, 0xc6671c10e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8c723c8a0, 0xc748056980, 0xc602083a90, 0xc6671c1198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8c723c8a0, 0xc5bfd51650, 0xc748056980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8c723c8c0, 0xc748056980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d5e0, 0xc97fade800, 0x740, 0x900, 0x598b438, 0x280bcc0, 0xc97fade800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d5e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 288 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x19872fe63fb43dd4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7eaa488e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7eaa488e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7eaa488e0, 0x28006b0, 0xc86e13a360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8606584a0, 0xc7eaa488e0, 0xc5cf6a5a90, 0xc86e13a418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8606584a0, 0xc5bfd516b0, 0xc7eaa488e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8606584e0, 0xc7eaa488e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d730, 0xc8cc105800, 0x7cd, 0x900, 0x2975bf4b, 0x280bcc0, 0xc8cc105800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 289 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4f57ac37776616d7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5552ef320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5552ef320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5552ef320, 0x28006b0, 0xc451d7d7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4e4286b40, 0xc5552ef320, 0xc5d5db5a90, 0xc451d7d858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4e4286b40, 0xc5bfd51710, 0xc5552ef320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4e4286b80, 0xc5552ef320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d880, 0xc747c0e000, 0xa38, 0xc00, 0x2c75f918, 0x280bcc0, 0xc747c0e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 290 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8433b328361c660e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc55526f080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc55526f080, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc55526f080, 0x28006b0, 0xc6ea654ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4b6f5fbc0, 0xc55526f080, 0xc5b871fa90, 0xc6ea654f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4b6f5fbc0, 0xc5bfd51770, 0xc55526f080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4b6f5fbe0, 0xc55526f080, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5d9d0, 0xc980dc3800, 0x702, 0x900, 0x19436aa1, 0x280bcc0, 0xc980dc3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5d9d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 291 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfb88fa31e09dfb49)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7ce2c8520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7ce2c8520, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7ce2c8520, 0x28006b0, 0xc737900480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f19f5040, 0xc7ce2c8520, 0xc5dfed3a90, 0xc737900538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f19f5040, 0xc5bfd517d0, 0xc7ce2c8520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f19f5060, 0xc7ce2c8520, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5db20, 0xc7c41b2000, 0x94e, 0xc00, 0x1ea6aaad, 0x280bcc0, 0xc7c41b2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5db20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 292 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdfca02da8fe24cdd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b1a07d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b1a07d80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b1a07d80, 0x28006b0, 0xc7e938c7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5e7391940, 0xc8b1a07d80, 0xc63d5c9a90, 0xc7e938c8a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5e7391940, 0xc5bfd51830, 0xc8b1a07d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5e7391960, 0xc8b1a07d80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5dc70, 0xc8b4e3b800, 0x900, 0x900, 0x1b27a589, 0x280bcc0, 0xc8b4e3b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5dc70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 293 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8a45b687c08521fb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc66335e780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc66335e780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc66335e780, 0x28006b0, 0xc986804fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc862aa3000, 0xc66335e780, 0xc59869ba90, 0xc986805078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc862aa3000, 0xc5bfd51890, 0xc66335e780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc862aa3020, 0xc66335e780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5ddc0, 0xc8cb3ca000, 0x9c9, 0xc00, 0x849ec96, 0x280bcc0, 0xc8cb3ca000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5ddc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 294 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcce7409eff1c7204)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc744cc2740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc744cc2740, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc744cc2740, 0x28006b0, 0xc86db34360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8696468c0, 0xc744cc2740, 0xc5b743fa90, 0xc86db34418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8696468c0, 0xc5bfd518f0, 0xc744cc2740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8696468e0, 0xc744cc2740, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5c5e5df10, 0xc986ad7000, 0x72e, 0x900, 0x1e5fc704, 0x280bcc0, 0xc986ad7000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5c5e5df10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 295 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9bacba01c1b365e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc830e2af20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc830e2af20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc830e2af20, 0x28006b0, 0xc56eaa57a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6a57b4a60, 0xc830e2af20, 0xc5adcbda90, 0xc56eaa5868)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6a57b4a60, 0xc5bfd51950, 0xc830e2af20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6a57b4a80, 0xc830e2af20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514070, 0xc7f84c8000, 0x798, 0x900, 0x2b87ba1d, 0x280bcc0, 0xc7f84c8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 296 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2cecce200afeedbb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5d05b7580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5d05b7580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5d05b7580, 0x28006b0, 0xc5ff402120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc44e7aa800, 0xc5d05b7580, 0xc6038f3a90, 0xc5ff4021d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc44e7aa800, 0xc5bfd519b0, 0xc5d05b7580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc44e7aa820, 0xc5d05b7580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d85141c0, 0xc8cbd7ea00, 0x536, 0x6a0, 0xba9885b, 0x280bcc0, 0xc8cbd7ea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d85141c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 297 [semacquire, 9419 minutes]:
sync.runtime_SemacquireMutex(0xc4206ef044)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ef040)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2d41737c4d26b01b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78c369820, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78c369820, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78c369820, 0x28006b0, 0xc7e445b320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8ba793200, 0xc78c369820, 0xc5b7b9fa90, 0xc7e445b3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8ba793200, 0xc5bfd51a10, 0xc78c369820, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8ba793260, 0xc78c369820, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514310, 0xc7af45d800, 0xc2, 0xc8, 0x694bfcc, 0x280bcc0, 0xc7af45d800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 298 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb23c739ab9a2f04a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc69a8430a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc69a8430a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc69a8430a0, 0x28006b0, 0xc5e6e5c5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc702518080, 0xc69a8430a0, 0xc5d358da90, 0xc5e6e5c658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc702518080, 0xc5bfd51a70, 0xc69a8430a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7025180a0, 0xc69a8430a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514460, 0xc7395a9000, 0x8dc, 0x900, 0x27899a43, 0x280bcc0, 0xc7395a9000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 299 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2db4fe590b906da8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e58ec2c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e58ec2c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e58ec2c0, 0x28006b0, 0xc76bd218c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54fbdaea0, 0xc6e58ec2c0, 0xc5dad65a90, 0xc76bd21978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54fbdaea0, 0xc5bfd51ad0, 0xc6e58ec2c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54fbdaec0, 0xc6e58ec2c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d85145b0, 0xc593602000, 0x48a, 0x6a0, 0x153835da, 0x280bcc0, 0xc593602000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d85145b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 300 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9306334dd47476dc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8cee0fe60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8cee0fe60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8cee0fe60, 0x28006b0, 0xc86cda1200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc73b009f00, 0xc8cee0fe60, 0xc5d14d3a90, 0xc86cda12b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc73b009f00, 0xc5bfd51b30, 0xc8cee0fe60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc73b009f60, 0xc8cee0fe60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514700, 0xc739830000, 0x853, 0x900, 0x27ec1d9, 0x280bcc0, 0xc739830000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 301 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdf1446b689d6a899)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74efdc980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74efdc980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74efdc980, 0x28006b0, 0xc79b623d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc840001b20, 0xc74efdc980, 0xc5d3e73a90, 0xc79b623de8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc840001b20, 0xc5bfd51b90, 0xc74efdc980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc840001b40, 0xc74efdc980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514850, 0xc7395a4800, 0x878, 0x900, 0x1292f393, 0x280bcc0, 0xc7395a4800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 302 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc35fc7e10b67b4e7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc764bc1d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc764bc1d20, 0x1a753c0, 0xc5d315ba18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc764bc1d20, 0x28006b0, 0xc806a14c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4dce08640, 0xc764bc1d20, 0xc5d315ba90, 0xc806a14d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4dce08640, 0xc5bfd51bf0, 0xc764bc1d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4dce086a0, 0xc764bc1d20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d85149a0, 0xc87186f500, 0x5ee, 0x6a0, 0x3aea3d63, 0x280bcc0, 0xc87186f500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d85149a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 303 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1cf16ee2f9065827)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88e3face0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88e3face0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88e3face0, 0x28006b0, 0xc7a3bb5560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc867d9d400, 0xc88e3face0, 0xc58452ba90, 0xc7a3bb5618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc867d9d400, 0xc5bfd51c50, 0xc88e3face0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc867d9d420, 0xc88e3face0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514af0, 0xc8b86e8000, 0x93b, 0xc00, 0x35d69fc7, 0x280bcc0, 0xc8b86e8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514af0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 304 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa3a85e2860b206bd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73dc9c500, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73dc9c500, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73dc9c500, 0x28006b0, 0xc6c9d938c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc558a12700, 0xc73dc9c500, 0xc5e3b4da90, 0xc6c9d93988)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc558a12700, 0xc5bfd51cb0, 0xc73dc9c500, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc558a12740, 0xc73dc9c500, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514c40, 0xc980dba800, 0x7d9, 0x900, 0x1ae38e11, 0x280bcc0, 0xc980dba800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514c40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 305 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x33e46b17343b6c8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7809db880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7809db880, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7809db880, 0x28006b0, 0xc80f7238c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5446eba00, 0xc7809db880, 0xc57e8c3a90, 0xc80f723978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5446eba00, 0xc5bfd51d10, 0xc7809db880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5446eba40, 0xc7809db880, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514d90, 0xc7f54d1800, 0x854, 0x900, 0xe3770d6, 0x280bcc0, 0xc7f54d1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514d90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 306 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x83acac1ebc0e3975)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6af070f60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6af070f60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6af070f60, 0x28006b0, 0xc67621c5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6cc1154c0, 0xc6af070f60, 0xc5840e3a90, 0xc67621c658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6cc1154c0, 0xc5bfd51d70, 0xc6af070f60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6cc115520, 0xc6af070f60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8514ee0, 0xc980721500, 0x643, 0x6a0, 0x24031c3, 0x280bcc0, 0xc980721500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8514ee0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 307 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa3da09b2bc3416bc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88a4b2ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88a4b2ae0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88a4b2ae0, 0x28006b0, 0xc7fe491200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6be694860, 0xc88a4b2ae0, 0xc63c40ba90, 0xc7fe4912b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6be694860, 0xc5bfd51dd0, 0xc88a4b2ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6be6948c0, 0xc88a4b2ae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515030, 0xc7d73e6800, 0x7c8, 0x900, 0x203a4af4, 0x280bcc0, 0xc7d73e6800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515030, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 308 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x96a5279d73b679ad)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6dcf6a1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6dcf6a1e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6dcf6a1e0, 0x28006b0, 0xc825733e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6a7e0f480, 0xc6dcf6a1e0, 0xc605b99a90, 0xc825733f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6a7e0f480, 0xc5bfd51e30, 0xc6dcf6a1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6a7e0f4a0, 0xc6dcf6a1e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515180, 0xc890be2000, 0x765, 0x900, 0x19d05f56, 0x280bcc0, 0xc890be2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515180, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 309 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x29d26237ec065fd4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5e733f940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5e733f940, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5e733f940, 0x28006b0, 0xc8c56bb200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ebb88c80, 0xc5e733f940, 0xc6032bfa90, 0xc8c56bb2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ebb88c80, 0xc5bfd51e90, 0xc5e733f940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ebb88cc0, 0xc5e733f940, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515340, 0xc86e482000, 0x9d0, 0xc00, 0xe9600ac, 0x280bcc0, 0xc86e482000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 310 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x408c48a28b8ccb4f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82ca57420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82ca57420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82ca57420, 0x28006b0, 0xc86d54afc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b8605420, 0xc82ca57420, 0xc63c44da90, 0xc86d54b078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b8605420, 0xc5bfd51ef0, 0xc82ca57420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b8605440, 0xc82ca57420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515490, 0xc7f6e68800, 0x807, 0x900, 0x1579bcf6, 0x280bcc0, 0xc7f6e68800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 311 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1fa953ff95afbe31)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc83c7d1380, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc83c7d1380, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc83c7d1380, 0x28006b0, 0xc5e72d6360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6faefaaa0, 0xc83c7d1380, 0xc5e32d9a90, 0xc5e72d6418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6faefaaa0, 0xc5bfd51f50, 0xc83c7d1380, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6faefaac0, 0xc83c7d1380, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d85155e0, 0xc7b38bf000, 0x84f, 0x900, 0x238f4107, 0x280bcc0, 0xc7b38bf000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d85155e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 312 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe4eea43a921661d5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc70fa67e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc70fa67e40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc70fa67e40, 0x28006b0, 0xc6376fea20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6f1439be0, 0xc70fa67e40, 0xc5b17efa90, 0xc6376fead8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6f1439be0, 0xc5bfd51fb0, 0xc70fa67e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6f1439c00, 0xc70fa67e40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515730, 0xc8cb3ac800, 0x6e4, 0x900, 0x1ddafaaf, 0x280bcc0, 0xc8cb3ac800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 313 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4d2ce22a4c55ad45)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5fa502b20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5fa502b20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5fa502b20, 0x28006b0, 0xc784df70e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b34154a0, 0xc5fa502b20, 0xc5a8af7a90, 0xc784df7198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b34154a0, 0xc5bf4ee030, 0xc5fa502b20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b34154c0, 0xc5fa502b20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515880, 0xc8bb920000, 0x985, 0xc00, 0x3478f87c, 0x280bcc0, 0xc8bb920000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 314 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe0c092bfdd0d2071)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc66b57a0a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc66b57a0a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc66b57a0a0, 0x28006b0, 0xc7f29fd200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7a472df40, 0xc66b57a0a0, 0xc5986a3a90, 0xc7f29fd2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7a472df40, 0xc5bf4ee090, 0xc66b57a0a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7a472df60, 0xc66b57a0a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d85159d0, 0xc7d1c02800, 0x845, 0x900, 0x18fa26fc, 0x280bcc0, 0xc7d1c02800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d85159d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 315 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9989f11fe8d7b82d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8c66391e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8c66391e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8c66391e0, 0x28006b0, 0xc67965f440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8470e7e20, 0xc8c66391e0, 0xc5d4a2da90, 0xc67965f4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8470e7e20, 0xc5bf4ee0f0, 0xc8c66391e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8470e7e40, 0xc8c66391e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515b20, 0xc8cb961800, 0x859, 0x900, 0x33e8484f, 0x280bcc0, 0xc8cb961800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515b20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 316 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe17cda5494b05685)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f1249ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f1249ea0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f1249ea0, 0x28006b0, 0xc88abe26c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc743a00280, 0xc7f1249ea0, 0xc5b10f5a90, 0xc88abe2778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc743a00280, 0xc5bf4ee150, 0xc7f1249ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc743a002a0, 0xc7f1249ea0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515c70, 0xc7b2661500, 0x645, 0x6a0, 0x3388e2bb, 0x280bcc0, 0xc7b2661500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515c70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 317 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x19b68010cad6f2a7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74bb57460, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74bb57460, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74bb57460, 0x28006b0, 0xc79e358b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81bb114a0, 0xc74bb57460, 0xc5b7f27a90, 0xc79e358bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81bb114a0, 0xc5bf4ee1b0, 0xc74bb57460, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81bb11540, 0xc74bb57460, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515dc0, 0xc890406000, 0x80e, 0x900, 0x33927dad, 0x280bcc0, 0xc890406000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515dc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 318 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5142367fef49b8d3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc738b4eca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc738b4eca0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc738b4eca0, 0x28006b0, 0xc54d3845a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc764fcdf60, 0xc738b4eca0, 0xc58662da90, 0xc54d384658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc764fcdf60, 0xc5bf4ee210, 0xc738b4eca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc764fcdf80, 0xc738b4eca0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d8515f10, 0xc8b179f000, 0x6eb, 0x900, 0x177793b9, 0x280bcc0, 0xc8b179f000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d8515f10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 319 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc354bdedc0a63288)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78fc5bbe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78fc5bbe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78fc5bbe0, 0x28006b0, 0xc7e4e42240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4c1cb6720, 0xc78fc5bbe0, 0xc5b0a51a90, 0xc7e4e422f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4c1cb6720, 0xc5bf4ee270, 0xc78fc5bbe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4c1cb6740, 0xc78fc5bbe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc070, 0xc9827c3800, 0x7d2, 0x900, 0xbe58513, 0x280bcc0, 0xc9827c3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 320 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6ee76973c26eeaf8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc796f9c920, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc796f9c920, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc796f9c920, 0x28006b0, 0xc865ba4a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc82cbb4b20, 0xc796f9c920, 0xc5b84d9a90, 0xc865ba4ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc82cbb4b20, 0xc5bf4ee2d0, 0xc796f9c920, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc82cbb4b40, 0xc796f9c920, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc1c0, 0xc7d760e800, 0x891, 0x900, 0x29927d0, 0x280bcc0, 0xc7d760e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc1c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 321 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8e85155aa025c286)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b3e136a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b3e136a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b3e136a0, 0x28006b0, 0xc7455c7e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7eee75080, 0xc7b3e136a0, 0xc606c6da90, 0xc7455c7f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7eee75080, 0xc5bf4ee330, 0xc7b3e136a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7eee750a0, 0xc7b3e136a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc310, 0xc7afca6000, 0xafc, 0xc00, 0x274c3598, 0x280bcc0, 0xc7afca6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 322 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x32ff08cf40caf583)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc682e25e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc682e25e00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc682e25e00, 0x28006b0, 0xc7f2b6bc20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc756825780, 0xc682e25e00, 0xc5abf6fa90, 0xc7f2b6bcd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc756825780, 0xc5bf4ee390, 0xc682e25e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7568257a0, 0xc682e25e00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc460, 0xc98679c800, 0x809, 0x900, 0x275f2609, 0x280bcc0, 0xc98679c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 323 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6bd498daea09d4fc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc563427c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc563427c60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc563427c60, 0x28006b0, 0xc59e62f200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70c80a120, 0xc563427c60, 0xc580133a90, 0xc59e62f2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70c80a120, 0xc5bf4ee3f0, 0xc563427c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70c80a140, 0xc563427c60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc5b0, 0xc7fb676000, 0x920, 0xc00, 0x2e38c4f4, 0x280bcc0, 0xc7fb676000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc5b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 324 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9ae9d50766fdb21f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8ca6f8580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8ca6f8580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8ca6f8580, 0x28006b0, 0xc836cf8900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc49787a500, 0xc8ca6f8580, 0xc5e0b95a90, 0xc836cf89c8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc49787a500, 0xc5bf4ee450, 0xc8ca6f8580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc49787a520, 0xc8ca6f8580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc700, 0xc7d6fd3000, 0x6e6, 0x900, 0x1eafa23c, 0x280bcc0, 0xc7d6fd3000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 325 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc10a846c4ce930e4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f3d8e460, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f3d8e460, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f3d8e460, 0x28006b0, 0xc5e4ef4120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7afc592e0, 0xc7f3d8e460, 0xc59f4d9a90, 0xc5e4ef41d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7afc592e0, 0xc5bf4ee4b0, 0xc7f3d8e460, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7afc59300, 0xc7f3d8e460, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efc8c0, 0xc8c5e94000, 0x98f, 0xc00, 0x34a8e13e, 0x280bcc0, 0xc8c5e94000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efc8c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 327 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe12107993e8c4a2f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7cab6a620, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7cab6a620, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7cab6a620, 0x28006b0, 0xc8c75fa480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5c3d23940, 0xc7cab6a620, 0xc605915a90, 0xc8c75fa538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5c3d23940, 0xc5bf4ee570, 0xc7cab6a620, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5c3d23980, 0xc7cab6a620, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efcb60, 0xc811142000, 0xa6d, 0xc00, 0x2084fd99, 0x280bcc0, 0xc811142000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efcb60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 328 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3a0359b6b6685e3b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6cdaa8b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6cdaa8b40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6cdaa8b40, 0x28006b0, 0xc803b5e360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4353d9f00, 0xc6cdaa8b40, 0xc5dc67ba90, 0xc803b5e418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4353d9f00, 0xc5bf4ee5d0, 0xc6cdaa8b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4353d9f20, 0xc6cdaa8b40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efccb0, 0xc7f8da1000, 0x7bf, 0x900, 0x24deb04d, 0x280bcc0, 0xc7f8da1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efccb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 329 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdd3248a91693b65b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc491673600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc491673600, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc491673600, 0x28006b0, 0xc4a3eaf440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5786a54c0, 0xc491673600, 0xc5dc6e5a90, 0xc4a3eaf4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5786a54c0, 0xc5bf4ee630, 0xc491673600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5786a54e0, 0xc491673600, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efce00, 0xc820a9a800, 0x7d6, 0x900, 0x29043b7a, 0x280bcc0, 0xc820a9a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efce00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 330 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x11708af621ccd4a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c6aeaa60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c6aeaa60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c6aeaa60, 0x28006b0, 0xc6c9cf4ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8305d62c0, 0xc7c6aeaa60, 0xc592379a90, 0xc6c9cf4f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8305d62c0, 0xc5bf4ee690, 0xc7c6aeaa60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8305d62e0, 0xc7c6aeaa60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efcf50, 0xc7b38c3800, 0x852, 0x900, 0xa9f6bec, 0x280bcc0, 0xc7b38c3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efcf50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 331 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc0201e2d3943e68b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc760306a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc760306a40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc760306a40, 0x28006b0, 0xc81e552a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7decc38e0, 0xc760306a40, 0xc605877a90, 0xc81e552ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7decc38e0, 0xc5bf4ee6f0, 0xc760306a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7decc3900, 0xc760306a40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd0a0, 0xc7eab58a00, 0x62e, 0x6a0, 0x13d9984e, 0x280bcc0, 0xc7eab58a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd0a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 332 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6815a7c164e2c2c6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8659bd9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8659bd9a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8659bd9a0, 0x28006b0, 0xc5ff140c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c7bb0b00, 0xc8659bd9a0, 0xc591d75a90, 0xc5ff140d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c7bb0b00, 0xc5bf4ee750, 0xc8659bd9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c7bb0b20, 0xc8659bd9a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd1f0, 0xc97fc70000, 0x7f7, 0x900, 0x25d88af1, 0x280bcc0, 0xc97fc70000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd1f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 333 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcfab39f454b86e76)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc762610680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc762610680, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc762610680, 0x28006b0, 0xc5b4fa0d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e505ed20, 0xc762610680, 0xc5aee33a90, 0xc5b4fa0e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e505ed20, 0xc5bf4ee7b0, 0xc762610680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e505ed60, 0xc762610680, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd340, 0xc8682b5800, 0x8f2, 0x900, 0x2f414460, 0x280bcc0, 0xc8682b5800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 334 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x383d1e4d44a2352c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6dbf0dc00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6dbf0dc00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6dbf0dc00, 0x28006b0, 0xc869f2a5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc600e8f800, 0xc6dbf0dc00, 0xc605dada90, 0xc869f2a658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc600e8f800, 0xc5bf4ee810, 0xc6dbf0dc00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc600e8f820, 0xc6dbf0dc00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd490, 0xc88ec16000, 0x807, 0x900, 0x26bbfb38, 0x280bcc0, 0xc88ec16000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 335 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4c6a099b85025fa3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7bbfbc9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7bbfbc9a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7bbfbc9a0, 0x28006b0, 0xc838092b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5bb17ea00, 0xc7bbfbc9a0, 0xc5e3af5a90, 0xc838092bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5bb17ea00, 0xc5bf4ee870, 0xc7bbfbc9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5bb17ea20, 0xc7bbfbc9a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd5e0, 0xc82c988000, 0x9a4, 0xc00, 0x1f1ca8f1, 0x280bcc0, 0xc82c988000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd5e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 336 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:440 +0x34b
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 337 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x149b2b1b9dd4490c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7de187bc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7de187bc0, 0x1a753c0, 0xc866d7f7c8)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7de187bc0, 0x28006b0, 0xc866d7f7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6e93dcca0, 0xc7de187bc0, 0xc5dee3ba90, 0xc866d7f858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6e93dcca0, 0xc5bf4ee930, 0xc7de187bc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6e93dccc0, 0xc7de187bc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd880, 0xc8682ac800, 0x878, 0x900, 0x3a12227f, 0x280bcc0, 0xc8682ac800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 338 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x73493dd13c16ee04)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f59db600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f59db600, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f59db600, 0x28006b0, 0xc787f258c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc48cbdfba0, 0xc6f59db600, 0xc5c02fba90, 0xc787f25978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc48cbdfba0, 0xc5bf4ee990, 0xc6f59db600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc48cbdfcc0, 0xc6f59db600, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efd9d0, 0xc7d0bee000, 0xa37, 0xc00, 0x2fdf31ed, 0x280bcc0, 0xc7d0bee000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efd9d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 339 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x399282905e49631a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc515a4fec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc515a4fec0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc515a4fec0, 0x28006b0, 0xc556e3e120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc564261e40, 0xc515a4fec0, 0xc5af7eba90, 0xc556e3e1e8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc564261e40, 0xc5bf4ee9f0, 0xc515a4fec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc564261e60, 0xc515a4fec0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efdb20, 0xc7d6fca000, 0x7a2, 0x900, 0x2397da68, 0x280bcc0, 0xc7d6fca000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efdb20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 340 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x28fb98dc9a6ee357)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4d46575c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4d46575c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4d46575c0, 0x28006b0, 0xc88ca71440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc46c1376a0, 0xc4d46575c0, 0xc63df09a90, 0xc88ca714f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc46c1376a0, 0xc5bf4eea50, 0xc4d46575c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc46c1376c0, 0xc4d46575c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efdc70, 0xc987679800, 0x843, 0x900, 0x3b023f27, 0x280bcc0, 0xc987679800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efdc70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 341 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x45b2f09fcb196445)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7353367a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7353367a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7353367a0, 0x28006b0, 0xc4c3e885a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc64c720500, 0xc7353367a0, 0xc5b59b1a90, 0xc4c3e88648)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc64c720500, 0xc5bf4eeab0, 0xc7353367a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc64c720540, 0xc7353367a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efddc0, 0xc7b6ca0a00, 0x40, 0xc8, 0xf03674c, 0x280bcc0, 0xc7b6ca0a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efddc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 342 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x78cd89689e3c40ad)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7aca0c780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7aca0c780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7aca0c780, 0x28006b0, 0xc6a2c1c240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7cf495bc0, 0xc7aca0c780, 0xc5989f5a90, 0xc6a2c1c308)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7cf495bc0, 0xc5bf4eeb10, 0xc7aca0c780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7cf495be0, 0xc7aca0c780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1efdf10, 0xc8c57dc000, 0x94d, 0xc00, 0x3b855a6f, 0x280bcc0, 0xc8c57dc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1efdf10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 343 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x76a9a3223adec19d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc860778900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc860778900, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc860778900, 0x28006b0, 0xc885ce66c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5852787a0, 0xc860778900, 0xc603b6da90, 0xc885ce6778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5852787a0, 0xc5bf4eeb70, 0xc860778900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5852787c0, 0xc860778900, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0070, 0xc739764800, 0x7f1, 0x900, 0xcabc478, 0x280bcc0, 0xc739764800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 344 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xad138324b3ebd736)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7054dcc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7054dcc40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7054dcc40, 0x28006b0, 0xc8adce58c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4cc7bdaa0, 0xc7054dcc40, 0xc5e395ba90, 0xc8adce5988)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4cc7bdaa0, 0xc5bf4eebd0, 0xc7054dcc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4cc7bdac0, 0xc7054dcc40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d01c0, 0xc85ef42000, 0xa5a, 0xc00, 0x1562b21c, 0x280bcc0, 0xc85ef42000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d01c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 345 [chan send, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).getOrCreateSeries(0xc4201dc000, 0x8cb84d73b546bfcc, 0xc6c35f49f0, 0x8cb84d73b546bfcc, 0xc420014000, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1025 +0x643
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6502bcd40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:908 +0x1bc
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6502bcd40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6502bcd40, 0x28006b0, 0xc809545200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5165588a0, 0xc6502bcd40, 0xc5b0097a90, 0xc8095452b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5165588a0, 0xc5bf4eec30, 0xc6502bcd40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5165588c0, 0xc6502bcd40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0310, 0xc8bbee7800, 0x747, 0x900, 0x1e46cb04, 0x280bcc0, 0xc8bbee7800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 346 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xec445edad903381a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8843ec060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8843ec060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8843ec060, 0x28006b0, 0xc684b3a240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8368f16a0, 0xc8843ec060, 0xc5ab53ba90, 0xc684b3a2f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8368f16a0, 0xc5bf4eec90, 0xc8843ec060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8368f16c0, 0xc8843ec060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0460, 0xc7f4a38a00, 0x5b7, 0x6a0, 0x33851aad, 0x280bcc0, 0xc7f4a38a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 347 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7e0fa9a795162496)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc462a91200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc462a91200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc462a91200, 0x28006b0, 0xc5c372a6c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c0b48d60, 0xc462a91200, 0xc5ac9b7a90, 0xc5c372a778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c0b48d60, 0xc5bf4eecf0, 0xc462a91200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c0b48d80, 0xc462a91200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d05b0, 0xc8b6312800, 0x784, 0x900, 0x1876bcbf, 0x280bcc0, 0xc8b6312800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d05b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 348 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd2d555a13e194a5d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e9e1a540, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e9e1a540, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e9e1a540, 0x28006b0, 0xc560b01200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc820605d20, 0xc6e9e1a540, 0xc5d75c1a90, 0xc560b012c8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc820605d20, 0xc5bf4eed50, 0xc6e9e1a540, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc820605d40, 0xc6e9e1a540, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0700, 0xc65a88e000, 0xa03, 0xc00, 0x33bd3278, 0x280bcc0, 0xc65a88e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 349 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb5354ed2e1a6295b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7606ade00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7606ade00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7606ade00, 0x28006b0, 0xc884dcf0e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6492671a0, 0xc7606ade00, 0xc602b83a90, 0xc884dcf1a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6492671a0, 0xc5bf4eedb0, 0xc7606ade00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6492671c0, 0xc7606ade00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0850, 0xc768b6b500, 0x59d, 0x6a0, 0x5f9f561, 0x280bcc0, 0xc768b6b500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 350 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xea1e2537ac6278b0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88673c520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88673c520, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88673c520, 0x28006b0, 0xc7f7a6b8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81d420c40, 0xc88673c520, 0xc5ab889a90, 0xc7f7a6b978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81d420c40, 0xc5bf4eee10, 0xc88673c520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81d420c80, 0xc88673c520, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d09a0, 0xc830218000, 0x93e, 0xc00, 0x3a9fbd45, 0x280bcc0, 0xc830218000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d09a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 351 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x51d1a6dc468117a4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86c435aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86c435aa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86c435aa0, 0x28006b0, 0xc771c698c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87a1679a0, 0xc86c435aa0, 0xc5a3e9da90, 0xc771c69978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87a1679a0, 0xc5bf4eee70, 0xc86c435aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87a1679e0, 0xc86c435aa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0af0, 0xc7effe4000, 0x978, 0xc00, 0x9f3f97, 0x280bcc0, 0xc7effe4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0af0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 352 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf6665a4e3f7ac21)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc683331ca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc683331ca0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc683331ca0, 0x28006b0, 0xc6d781e900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d3485bc0, 0xc683331ca0, 0xc59f4d7a90, 0xc6d781e9b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d3485bc0, 0xc5bf4eeed0, 0xc683331ca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d3485c40, 0xc683331ca0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0c40, 0xc8bb558000, 0x82d, 0x900, 0x19ae39eb, 0x280bcc0, 0xc8bb558000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0c40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 353 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x84b17957adfd454f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc703769cc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc703769cc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc703769cc0, 0x28006b0, 0xc8336226c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7562c6ce0, 0xc703769cc0, 0xc825da9a90, 0xc833622778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7562c6ce0, 0xc5bf4eef30, 0xc703769cc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7562c6d00, 0xc703769cc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0e00, 0xc835584000, 0x937, 0xc00, 0x1eed19c3, 0x280bcc0, 0xc835584000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0e00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 354 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x98d0b21c76f64b29)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc68fd66580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc68fd66580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc68fd66580, 0x28006b0, 0xc87bba5680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc88ac710a0, 0xc68fd66580, 0xc5e264da90, 0xc87bba5738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc88ac710a0, 0xc5bf4eef90, 0xc68fd66580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc88ac710c0, 0xc68fd66580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d0f50, 0xc7a76de000, 0xb39, 0xc00, 0x2f647bfa, 0x280bcc0, 0xc7a76de000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d0f50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 355 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9f7f8b35a8a6dfa)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc762810ca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc762810ca0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc762810ca0, 0x28006b0, 0xc7c2d099e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86a8ad680, 0xc762810ca0, 0xc5b0503a90, 0xc7c2d09a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86a8ad680, 0xc5bf4eeff0, 0xc762810ca0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86a8ad6a0, 0xc762810ca0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d10a0, 0xc7d65a0000, 0x9a1, 0xc00, 0x19406e2a, 0x280bcc0, 0xc7d65a0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d10a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 356 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xadb5d2f90e4ed6c5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73e2246c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73e2246c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73e2246c0, 0x28006b0, 0xc835522360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70bee3fc0, 0xc73e2246c0, 0xc59cb69a90, 0xc835522418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70bee3fc0, 0xc5bf4ef050, 0xc73e2246c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70bee3fe0, 0xc73e2246c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d11f0, 0xc7f3270000, 0x76d, 0x900, 0x10920fcc, 0x280bcc0, 0xc7f3270000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d11f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 357 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xae092b0d3ef95d95)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f1a47bc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f1a47bc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f1a47bc0, 0x28006b0, 0xc7836ad320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87dd243c0, 0xc7f1a47bc0, 0xc5986bba90, 0xc7836ad3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87dd243c0, 0xc5bf4ef0b0, 0xc7f1a47bc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87dd243e0, 0xc7f1a47bc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1340, 0xc858484000, 0x8dc, 0x900, 0x1dafe233, 0x280bcc0, 0xc858484000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 358 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcf1115348ce26833)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc60335c280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc60335c280, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc60335c280, 0x28006b0, 0xc753a758c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7624f9740, 0xc60335c280, 0xc606c67a90, 0xc753a75978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7624f9740, 0xc5bf4ef110, 0xc60335c280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7624f9760, 0xc60335c280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1490, 0xc7f759a000, 0xa2c, 0xc00, 0x34a212af, 0x280bcc0, 0xc7f759a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 359 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf711747987be20fd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc826a64f80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc826a64f80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc826a64f80, 0x28006b0, 0xc6b31e4480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7caf28ea0, 0xc826a64f80, 0xc594623a90, 0xc6b31e4538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7caf28ea0, 0xc5bf4ef170, 0xc826a64f80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7caf28ec0, 0xc826a64f80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d15e0, 0xc845102000, 0x8a2, 0x900, 0x1ff754a3, 0x280bcc0, 0xc845102000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d15e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 360 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd9b1205e12fb1ac4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b21e5f40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b21e5f40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b21e5f40, 0x28006b0, 0xc72eb62ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc840dd0020, 0xc8b21e5f40, 0xc5db7a7a90, 0xc72eb62f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc840dd0020, 0xc5bf4ef1d0, 0xc8b21e5f40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc840dd0040, 0xc8b21e5f40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1730, 0xc80991a000, 0x689, 0x6a0, 0x2ec6723c, 0x280bcc0, 0xc80991a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 361 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xac6736ce288c3fa7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8394e5e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8394e5e60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8394e5e60, 0x28006b0, 0xc7c1791320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc856ad0b00, 0xc8394e5e60, 0xc5d9585a90, 0xc7c17913d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc856ad0b00, 0xc5bf4ef230, 0xc8394e5e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc856ad0b20, 0xc8394e5e60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1880, 0xc7cff47000, 0x78e, 0x900, 0x21226133, 0x280bcc0, 0xc7cff47000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 362 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4369180e660571c8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c8ae7c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c8ae7c60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c8ae7c60, 0x28006b0, 0xc4ca46a240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8375a9b20, 0xc7c8ae7c60, 0xc5d9d51a90, 0xc4ca46a2f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8375a9b20, 0xc5bf4ef290, 0xc7c8ae7c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8375a9b40, 0xc7c8ae7c60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d19d0, 0xc82a000000, 0x853, 0x900, 0x2a583e4b, 0x280bcc0, 0xc82a000000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d19d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 363 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcd6a8c25a2e04dff)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4d78d32a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4d78d32a0, 0x1a753c0, 0xc5c064da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4d78d32a0, 0x28006b0, 0xc461863e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc71cce0b40, 0xc4d78d32a0, 0xc5c064da90, 0xc461863f28)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc71cce0b40, 0xc5bf4ef2f0, 0xc4d78d32a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc71cce0b80, 0xc4d78d32a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1b20, 0xc874384000, 0x65f, 0x6a0, 0x23ab5d9d, 0x280bcc0, 0xc874384000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1b20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 364 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xee957e1dabe7941f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6eee609c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6eee609c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6eee609c0, 0x28006b0, 0xc7c68e6120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc694738540, 0xc6eee609c0, 0xc602edba90, 0xc7c68e61d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc694738540, 0xc5bf4ef350, 0xc6eee609c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc694738580, 0xc6eee609c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1c70, 0xc81dc68800, 0x7a6, 0x900, 0x2730936a, 0x280bcc0, 0xc81dc68800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1c70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 365 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6e157e67aa8b4e9e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc76d5c6dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc76d5c6dc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc76d5c6dc0, 0x28006b0, 0xc8629dde60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c625b9c0, 0xc76d5c6dc0, 0xc59b627a90, 0xc8629ddf18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c625b9c0, 0xc5bf4ef3b0, 0xc76d5c6dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c625b9e0, 0xc76d5c6dc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1dc0, 0xc844d4e800, 0x830, 0x900, 0xce8b79d, 0x280bcc0, 0xc844d4e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1dc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 366 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2025b7bf77dd7358)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74cc39b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74cc39b80, 0x1a753c0, 0xc5ae32fa18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74cc39b80, 0x28006b0, 0xc7cdea9560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8c9596520, 0xc74cc39b80, 0xc5ae32fa90, 0xc7cdea9618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8c9596520, 0xc5bf4ef410, 0xc74cc39b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8c9596560, 0xc74cc39b80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35d1f10, 0xc7aa830000, 0x7d3, 0x900, 0x1c37d0de, 0x280bcc0, 0xc7aa830000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35d1f10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 367 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa6785fdcd1e89613)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc516d05300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc516d05300, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc516d05300, 0x28006b0, 0xc86e891200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc68e42ae20, 0xc516d05300, 0xc586d77a90, 0xc86e8912b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc68e42ae20, 0xc5bf4ef470, 0xc516d05300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc68e42ae40, 0xc516d05300, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e0070, 0xc8906b0000, 0x749, 0x900, 0x1d1ed1ec, 0x280bcc0, 0xc8906b0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e0070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 368 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x880618f8acc19bfa)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc712ff0320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc712ff0320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc712ff0320, 0x28006b0, 0xc5e08af560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81c7545a0, 0xc712ff0320, 0xc5ce6a1a90, 0xc5e08af618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81c7545a0, 0xc5bf4ef4d0, 0xc712ff0320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81c7545c0, 0xc712ff0320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e01c0, 0xc7c612e000, 0x9a1, 0xc00, 0x3ab88b64, 0x280bcc0, 0xc7c612e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e01c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 369 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x480fab12ec0dcf7e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc772e52440, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc772e52440, 0x1a753c0, 0xc5ac31da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc772e52440, 0x28006b0, 0xc69ab7bb00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8778406a0, 0xc772e52440, 0xc5ac31da90, 0xc69ab7bbb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8778406a0, 0xc5bf4ef530, 0xc772e52440, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8778406c0, 0xc772e52440, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e0310, 0xc987a9c800, 0x879, 0x900, 0x3a44a15f, 0x280bcc0, 0xc987a9c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e0310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 370 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd7aa89c4df44a5ba)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e45c8ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e45c8ec0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e45c8ec0, 0x28006b0, 0xc59bd8cea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc836a0e580, 0xc4e45c8ec0, 0xc604197a90, 0xc59bd8cf58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc836a0e580, 0xc5bf4ef590, 0xc4e45c8ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc836a0e5c0, 0xc4e45c8ec0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e0460, 0xc82bd38000, 0x983, 0xc00, 0x2bfba362, 0x280bcc0, 0xc82bd38000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e0460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 371 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x38e6e997e31e954a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc784792040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc784792040, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc784792040, 0x28006b0, 0xc7c09025a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c8488520, 0xc784792040, 0xc5a2595a90, 0xc7c0902658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c8488520, 0xc5bf4ef5f0, 0xc784792040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c8488540, 0xc784792040, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e05b0, 0xc8144d2000, 0x91e, 0xc00, 0x258c6611, 0x280bcc0, 0xc8144d2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e05b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 372 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x37856e663be9a6b5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc764746760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc764746760, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc764746760, 0x28006b0, 0xc780643b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7f214f0c0, 0xc764746760, 0xc601ff3a90, 0xc780643bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7f214f0c0, 0xc5bf4ef650, 0xc764746760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7f214f100, 0xc764746760, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e0700, 0xc846891000, 0x786, 0x900, 0x1eca36b4, 0x280bcc0, 0xc846891000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e0700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 373 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6959ac5f1708c370)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a20cb480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a20cb480, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a20cb480, 0x28006b0, 0xc65e268120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc790a7f740, 0xc7a20cb480, 0xc5ad271a90, 0xc65e2681e8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc790a7f740, 0xc5bf4ef6b0, 0xc7a20cb480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc790a7f760, 0xc7a20cb480, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e35e0850, 0xc7fa032800, 0x7b0, 0x900, 0x17615ebf, 0x280bcc0, 0xc7fa032800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e35e0850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 260 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4274fcc46afeae5b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5fa1e87e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5fa1e87e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5fa1e87e0, 0x28006b0, 0xc65291b0e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5a7cde2c0, 0xc5fa1e87e0, 0xc5bab2fa90, 0xc65291b198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5a7cde2c0, 0xc5c4c80750, 0xc5fa1e87e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5a7cde2e0, 0xc5fa1e87e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771730, 0xc7d4e10000, 0x8d8, 0x900, 0x3490e1b1, 0x280bcc0, 0xc7d4e10000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 261 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc35a4e42b58343a0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6decca280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6decca280, 0x1a753c0, 0xc592391a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6decca280, 0x28006b0, 0xc44e87fb00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7cb9b1360, 0xc6decca280, 0xc592391a90, 0xc44e87fbb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7cb9b1360, 0xc5c4c807b0, 0xc6decca280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7cb9b1380, 0xc6decca280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771880, 0xc8baf68000, 0x778, 0x900, 0x26edf2c8, 0x280bcc0, 0xc8baf68000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 262 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8a043560fa42b2a4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e354d380, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e354d380, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e354d380, 0x28006b0, 0xc586a319e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7051328c0, 0xc6e354d380, 0xc5a4a5da90, 0xc586a31a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7051328c0, 0xc5c4c80810, 0xc6e354d380, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7051328e0, 0xc6e354d380, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac7719d0, 0xc97bf62000, 0x747, 0x900, 0x13b1a98f, 0x280bcc0, 0xc97bf62000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac7719d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 263 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x65df26646c69c5ce)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86ca70ba0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86ca70ba0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86ca70ba0, 0x28006b0, 0xc840de6b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7081fa880, 0xc86ca70ba0, 0xc5c4c9ba90, 0xc840de6bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7081fa880, 0xc5c4c80870, 0xc86ca70ba0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7081fa8c0, 0xc86ca70ba0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771b20, 0xc8bb474000, 0x722, 0x900, 0x2866d730, 0x280bcc0, 0xc8bb474000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771b20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 264 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x719adf9572def288)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc638b66520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc638b66520, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc638b66520, 0x28006b0, 0xc639dab440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4def13ae0, 0xc638b66520, 0xc593ab9a90, 0xc639dab4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4def13ae0, 0xc5c4c808d0, 0xc638b66520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4def13b00, 0xc638b66520, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771c70, 0xc8924ec000, 0xa01, 0xc00, 0x3defd63, 0x280bcc0, 0xc8924ec000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771c70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 265 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc70cb19e954ffb12)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6b14d10c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6b14d10c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6b14d10c0, 0x28006b0, 0xc7ed90c5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc56d8d4b60, 0xc6b14d10c0, 0xc580055a90, 0xc7ed90c668)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc56d8d4b60, 0xc5c4c80930, 0xc6b14d10c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc56d8d4ba0, 0xc6b14d10c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771dc0, 0xc7d6584000, 0x7ce, 0x900, 0x8482f82, 0x280bcc0, 0xc7d6584000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771dc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 266 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8a92e15bcbc582b9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a6999420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a6999420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a6999420, 0x28006b0, 0xc70e89f8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc73c97c540, 0xc7a6999420, 0xc5dee3fa90, 0xc70e89f978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc73c97c540, 0xc5c4c80990, 0xc7a6999420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc73c97c560, 0xc7a6999420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ac771f10, 0xc7ea2c6000, 0x961, 0xc00, 0x1d7b0bfa, 0x280bcc0, 0xc7ea2c6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ac771f10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 267 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2de6d6495d56e27a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c5f51d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c5f51d00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c5f51d00, 0x28006b0, 0xc7bec62d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ae685f20, 0xc7c5f51d00, 0xc63476fa90, 0xc7bec62e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ae685f20, 0xc5c4c809f0, 0xc7c5f51d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7ae685f40, 0xc7c5f51d00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e070, 0xc7f9a1e800, 0x8fb, 0x900, 0x20b45f98, 0x280bcc0, 0xc7f9a1e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 268 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x272937b06c5f5a0a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c1abb880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c1abb880, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c1abb880, 0x28006b0, 0xc87f2aa000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8873d6320, 0xc7c1abb880, 0xc5aee4ba90, 0xc87f2aa0b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8873d6320, 0xc5c4c80a50, 0xc7c1abb880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8873d6340, 0xc7c1abb880, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e1c0, 0xc8cc0fc800, 0x6cc, 0x900, 0x23645e2d, 0x280bcc0, 0xc8cc0fc800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e1c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 269 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4aca70a321ec70aa)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81c9eb9e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81c9eb9e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81c9eb9e0, 0x28006b0, 0xc5092bdc20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c2b577a0, 0xc81c9eb9e0, 0xc596593a90, 0xc5092bdcd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c2b577a0, 0xc5c4c80ab0, 0xc81c9eb9e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c2b577c0, 0xc81c9eb9e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e310, 0xc8cc504800, 0x833, 0x900, 0x25bd1d7a, 0x280bcc0, 0xc8cc504800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 270 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x213393e3c2bf4b2e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc75e2e2e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc75e2e2e60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc75e2e2e60, 0x28006b0, 0xc5b6738360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6f104db40, 0xc75e2e2e60, 0xc598199a90, 0xc5b6738418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6f104db40, 0xc5c4c80b10, 0xc75e2e2e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6f104db60, 0xc75e2e2e60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e460, 0xc81c642000, 0xa79, 0xc00, 0xc69db4f, 0x280bcc0, 0xc81c642000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 271 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4ec7bd64105eacdf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6cd082180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6cd082180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6cd082180, 0x28006b0, 0xc6e54dc900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6b41be840, 0xc6cd082180, 0xc5d3b69a90, 0xc6e54dc9b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6b41be840, 0xc5c4c80b70, 0xc6cd082180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6b41be860, 0xc6cd082180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e5b0, 0xc8ae4a6000, 0xba3, 0xc00, 0x10e38ce4, 0x280bcc0, 0xc8ae4a6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e5b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 272 [chan send, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).getOrCreateSeries(0xc4201dc000, 0xdc8df38497044906, 0xc75b81a900, 0xdc8df38497044906, 0xc42001a000, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1025 +0x643
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8473c4320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:908 +0x1bc
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8473c4320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8473c4320, 0x28006b0, 0xc839f4e7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86a105560, 0xc8473c4320, 0xc606fd5a90, 0xc839f4e898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86a105560, 0xc5c4c80bd0, 0xc8473c4320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86a1055c0, 0xc8473c4320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e700, 0xc7f9a31500, 0x508, 0x6a0, 0x163bf175, 0x280bcc0, 0xc7f9a31500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 385 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6bf2189009f51ce3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc76073c1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc76073c1c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc76073c1c0, 0x28006b0, 0xc7b9bdbd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ee609740, 0xc76073c1c0, 0xc602625a90, 0xc7b9bdbdf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ee609740, 0xc5c4c80c30, 0xc76073c1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ee609760, 0xc76073c1c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e850, 0xc7eb34c000, 0x995, 0xc00, 0x231732ec, 0x280bcc0, 0xc7eb34c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 386 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x593c0d1206677314)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c5bb2780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c5bb2780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c5bb2780, 0x28006b0, 0xc52c122a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80acf2ee0, 0xc7c5bb2780, 0xc5ad275a90, 0xc52c122ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80acf2ee0, 0xc5c4c80c90, 0xc7c5bb2780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80acf2f00, 0xc7c5bb2780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280e9a0, 0xc8c4af8000, 0xb55, 0xc00, 0x2e3f81aa, 0x280bcc0, 0xc8c4af8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280e9a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 387 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x10411715fec5a613)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7cba94860, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7cba94860, 0x1a753c0, 0xc56cb91448)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7cba94860, 0x28006b0, 0xc56cb91440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc837223d60, 0xc7cba94860, 0xc5fc451a90, 0xc56cb914f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc837223d60, 0xc5c4c80cf0, 0xc7cba94860, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc837223d80, 0xc7cba94860, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280eaf0, 0xc981db4800, 0x7cb, 0x900, 0x1ac6f1c, 0x280bcc0, 0xc981db4800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280eaf0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 388 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x77f1fd97ee557be5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc698745e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc698745e40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc698745e40, 0x28006b0, 0xc8ccd85680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4af324b40, 0xc698745e40, 0xc58658da90, 0xc8ccd85738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4af324b40, 0xc5c4c80d50, 0xc698745e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4af324b80, 0xc698745e40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280ec40, 0xc7d73eb000, 0x8a4, 0x900, 0x3a6cd2e8, 0x280bcc0, 0xc7d73eb000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280ec40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 389 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb9654b38e19e3d5f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc45c239f20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc45c239f20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc45c239f20, 0x28006b0, 0xc7248670e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc824e2c140, 0xc45c239f20, 0xc5dfed5a90, 0xc724867198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc824e2c140, 0xc5c4c80db0, 0xc45c239f20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc824e2c160, 0xc45c239f20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280ed90, 0xc987454000, 0x8d1, 0x900, 0x198b8bb0, 0x280bcc0, 0xc987454000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280ed90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 390 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x156683389c7fcc7d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4d551bce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4d551bce0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4d551bce0, 0x28006b0, 0xc816a81e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc629cf9100, 0xc4d551bce0, 0xc5967b3a90, 0xc816a81f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc629cf9100, 0xc5c4c80e10, 0xc4d551bce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc629cf9120, 0xc4d551bce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280eee0, 0xc890413800, 0x837, 0x900, 0x10a1d10c, 0x280bcc0, 0xc890413800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280eee0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 391 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xac7c31afedd76f42)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b9ab2aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b9ab2aa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b9ab2aa0, 0x28006b0, 0xc660d2ab40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc64a640440, 0xc8b9ab2aa0, 0xc5b0927a90, 0xc660d2abf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc64a640440, 0xc5c4c80e70, 0xc8b9ab2aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc64a640460, 0xc8b9ab2aa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f030, 0xc601265800, 0x878, 0x900, 0x11f0da51, 0x280bcc0, 0xc601265800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f030, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 392 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x85aa10692ae082e8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8574faee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8574faee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8574faee0, 0x28006b0, 0xc8cd7ff8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8614e2620, 0xc8574faee0, 0xc5c050da90, 0xc8cd7ff978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8614e2620, 0xc5c4c80ed0, 0xc8574faee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8614e2640, 0xc8574faee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f180, 0xc84ba40a00, 0x668, 0x6a0, 0x9e1e9b8, 0x280bcc0, 0xc84ba40a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f180, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 393 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xbd15860801e4f609)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7dfb74140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7dfb74140, 0x1a753c0, 0xc5dc90ee00)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7dfb74140, 0x28006b0, 0xc5b54c4240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc482144320, 0xc7dfb74140, 0xc5d905ba90, 0xc5b54c42f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc482144320, 0xc5c4c80f30, 0xc7dfb74140, 0xc482144340, 0x1a00660)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc482144340, 0xc7dfb74140, 0x27535e0, 0xc482144340)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f2d0, 0xc7ea93c800, 0x867, 0x900, 0x19a7ecf8, 0x280bcc0, 0xc7ea93c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f2d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 394 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6c4cb236d238e61)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc55ccc5480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc55ccc5480, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc55ccc5480, 0x28006b0, 0xc772d44000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ae814f60, 0xc55ccc5480, 0xc602969a90, 0xc772d440b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ae814f60, 0xc5c4c80f90, 0xc55ccc5480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ae814f80, 0xc55ccc5480, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f420, 0xc9872d6000, 0x767, 0x900, 0x1ef606e2, 0x280bcc0, 0xc9872d6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f420, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 395 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb9b1016f645ae31a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8cddfcfc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8cddfcfc0, 0x1a753c0, 0xc63476da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8cddfcfc0, 0x28006b0, 0xc4a56f58c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc823c49520, 0xc8cddfcfc0, 0xc63476da90, 0xc4a56f5988)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc823c49520, 0xc5c4c80ff0, 0xc8cddfcfc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc823c49540, 0xc8cddfcfc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f570, 0xc980167000, 0x8e0, 0x900, 0x1391fe87, 0x280bcc0, 0xc980167000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f570, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 396 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf625ebc7cf10b532)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc80c7ce600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc80c7ce600, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc80c7ce600, 0x28006b0, 0xc5b4799560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc51aef5da0, 0xc80c7ce600, 0xc596c23a90, 0xc5b4799628)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc51aef5da0, 0xc5c4c81050, 0xc80c7ce600, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc51aef5dc0, 0xc80c7ce600, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f6c0, 0xc7d5b89000, 0x861, 0x900, 0x506e68c, 0x280bcc0, 0xc7d5b89000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f6c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 397 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8e42bba0cbfc87a5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7d1781a60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7d1781a60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7d1781a60, 0x28006b0, 0xc6c816d200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5ccf8d360, 0xc7d1781a60, 0xc5ab885a90, 0xc6c816d2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5ccf8d360, 0xc5c4c810b0, 0xc7d1781a60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5ccf8d380, 0xc7d1781a60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f880, 0xc7f3706000, 0xa3f, 0xc00, 0x8771066, 0x280bcc0, 0xc7f3706000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 398 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa0da81f21b7162e2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5677c8400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5677c8400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5677c8400, 0x28006b0, 0xc586ec70e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc735966080, 0xc5677c8400, 0xc57fc07a90, 0xc586ec7198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc735966080, 0xc5c4c81110, 0xc5677c8400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7359660a0, 0xc5677c8400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280f9d0, 0xc81f61c000, 0xa1b, 0xc00, 0xf6a9fa3, 0x280bcc0, 0xc81f61c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280f9d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 399 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb3fee96501f6f1f6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e361a620, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e361a620, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e361a620, 0x28006b0, 0xc8622727e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87104cb60, 0xc7e361a620, 0xc601921a90, 0xc862272898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87104cb60, 0xc5c4c81170, 0xc7e361a620, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87104cb80, 0xc7e361a620, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280fb20, 0xc8916f9000, 0x8c3, 0x900, 0x1e5c9e08, 0x280bcc0, 0xc8916f9000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280fb20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 400 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcf11df12fa511f31)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74042f3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74042f3e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74042f3e0, 0x28006b0, 0xc873454ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d3c8f420, 0xc74042f3e0, 0xc605187a90, 0xc873454f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d3c8f420, 0xc5c4c811d0, 0xc74042f3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d3c8f460, 0xc74042f3e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280fce0, 0xc8ba81e000, 0x7d5, 0x900, 0xda4b47f, 0x280bcc0, 0xc8ba81e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280fce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 401 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcb48b7b87a9f12e4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc75f05f180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc75f05f180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc75f05f180, 0x28006b0, 0xc7a8186360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6f83bc240, 0xc75f05f180, 0xc5da8fda90, 0xc7a8186418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6f83bc240, 0xc5c4c81230, 0xc75f05f180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6f83bc260, 0xc75f05f180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280fe30, 0xc848d48800, 0x73e, 0x900, 0x38083901, 0x280bcc0, 0xc848d48800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280fe30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 402 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x20627a87bbf7acf6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e08a7840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e08a7840, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e08a7840, 0x28006b0, 0xc82947cc60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc890ab4a40, 0xc7e08a7840, 0xc6385a3a90, 0xc82947cd18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc890ab4a40, 0xc5c4c81290, 0xc7e08a7840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc890ab4a60, 0xc7e08a7840, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e280ff80, 0xc8b75c8800, 0x70b, 0x900, 0x36711456, 0x280bcc0, 0xc8b75c8800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e280ff80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 403 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x41bd742118f37b2b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f012b280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f012b280, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f012b280, 0x28006b0, 0xc844cd6b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e0fdcde0, 0xc7f012b280, 0xc5ad767a90, 0xc844cd6bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e0fdcde0, 0xc5c4c812f0, 0xc7f012b280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e0fdce00, 0xc7f012b280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe0e0, 0xc8c8efe000, 0x788, 0x900, 0x157e11ba, 0x280bcc0, 0xc8c8efe000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 404 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd1b83552ce06aab1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56bc09ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56bc09ae0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56bc09ae0, 0x28006b0, 0xc7c081c240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4e4e1aaa0, 0xc56bc09ae0, 0xc5dacc5a90, 0xc7c081c2f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4e4e1aaa0, 0xc5c4c81350, 0xc56bc09ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4e4e1aae0, 0xc56bc09ae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe230, 0xc7a26e1000, 0x7ef, 0x900, 0x119dd438, 0x280bcc0, 0xc7a26e1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 405 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8ef4fac0c2cd265)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a95a57e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a95a57e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a95a57e0, 0x28006b0, 0xc84a60ab40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7fe220160, 0xc7a95a57e0, 0xc5da1afa90, 0xc84a60abf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7fe220160, 0xc5c4c813b0, 0xc7a95a57e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7fe220180, 0xc7a95a57e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe380, 0xc7398d1000, 0x8fa, 0x900, 0x1298d7b4, 0x280bcc0, 0xc7398d1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 406 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xba421503b42ddeed)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc622172b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc622172b00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc622172b00, 0x28006b0, 0xc69be2a6c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc554e991e0, 0xc622172b00, 0xc586b0ba90, 0xc69be2a778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc554e991e0, 0xc5c4c81410, 0xc622172b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc554e99240, 0xc622172b00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe4d0, 0xc85f842000, 0x7c7, 0x900, 0x2719b6f2, 0x280bcc0, 0xc85f842000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe4d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 407 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa6b55663dc860097)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b12d4da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b12d4da0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b12d4da0, 0x28006b0, 0xc886750d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc571892ac0, 0xc7b12d4da0, 0xc603fe5a90, 0xc886750e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc571892ac0, 0xc5c4c81470, 0xc7b12d4da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc571892ae0, 0xc7b12d4da0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe620, 0xc7d4e1d800, 0x75a, 0x900, 0x4725d6, 0x280bcc0, 0xc7d4e1d800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 408 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6aff9380d9f41215)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc876197740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc876197740, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc876197740, 0x28006b0, 0xc4c98e5b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc69fb77b80, 0xc876197740, 0xc591d79a90, 0xc4c98e5bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc69fb77b80, 0xc5c4c814d0, 0xc876197740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc69fb77ba0, 0xc876197740, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe770, 0xc8bb561000, 0x884, 0x900, 0x39313079, 0x280bcc0, 0xc8bb561000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 409 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8a2f28ad416e6826)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc55796d340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc55796d340, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc55796d340, 0x28006b0, 0xc5843379e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6a0100740, 0xc55796d340, 0xc639115a90, 0xc584337a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6a0100740, 0xc5c4c81530, 0xc55796d340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6a0100760, 0xc55796d340, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fe8c0, 0xc864edc000, 0x9f8, 0xc00, 0x1f23472a, 0x280bcc0, 0xc864edc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fe8c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 410 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8964877d1dca5290)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc793814ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc793814ce0, 0x1a753c0, 0xc5aed23a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc793814ce0, 0x28006b0, 0xc70b010360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc79382cde0, 0xc793814ce0, 0xc5aed23a90, 0xc70b010428)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc79382cde0, 0xc5c4c81590, 0xc793814ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc79382ce00, 0xc793814ce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fea10, 0xc987cbc000, 0x499, 0x6a0, 0x2ee7ec2e, 0x280bcc0, 0xc987cbc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fea10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 411 [chan send, 10610 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).getOrCreateSeries(0xc4201dc000, 0x45e9856f1c10501a, 0xc5e34ade60, 0x45e9856f1c10501a, 0xc420016600, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1025 +0x643
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc75608da40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:908 +0x1bc
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc75608da40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc75608da40, 0x28006b0, 0xc5b8cce120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8650b88e0, 0xc75608da40, 0xc5b10f3a90, 0xc5b8cce1d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8650b88e0, 0xc5c4c815f0, 0xc75608da40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8650b8900, 0xc75608da40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56feb60, 0xc7fa41a000, 0x5b2, 0x6a0, 0x22539a3b, 0x280bcc0, 0xc7fa41a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56feb60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 412 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe4650ba930a7236f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc72bc4dd00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc72bc4dd00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc72bc4dd00, 0x28006b0, 0xc7e9659440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c16e5260, 0xc72bc4dd00, 0xc5d7d19a90, 0xc7e96594f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c16e5260, 0xc5c4c81650, 0xc72bc4dd00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c16e5280, 0xc72bc4dd00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fecb0, 0xc9827bf000, 0x858, 0x900, 0x60994bc, 0x280bcc0, 0xc9827bf000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fecb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 413 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3e5989a65ca372fc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc694388c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc694388c60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc694388c60, 0x28006b0, 0xc786e99560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6584d0600, 0xc694388c60, 0xc5b681fa90, 0xc786e99618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6584d0600, 0xc5c4c816b0, 0xc694388c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6584d0620, 0xc694388c60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fee00, 0xc8c624a000, 0x412, 0x6a0, 0x23dd1a35, 0x280bcc0, 0xc8c624a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fee00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 414 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4b50b8c3404ad0e8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7d0864520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7d0864520, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7d0864520, 0x28006b0, 0xc870a4db00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b376be40, 0xc7d0864520, 0xc598d1fa90, 0xc870a4dbb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b376be40, 0xc5c4c81710, 0xc7d0864520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b376be80, 0xc7d0864520, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fefc0, 0xc8a9188000, 0x7d5, 0x900, 0x1fb6c565, 0x280bcc0, 0xc8a9188000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fefc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 415 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x582af0c4ee91b196)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8ce6a92c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8ce6a92c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8ce6a92c0, 0x28006b0, 0xc45b701440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8a99040c0, 0xc8ce6a92c0, 0xc5d85dba90, 0xc45b7014f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8a99040c0, 0xc5c4c81770, 0xc8ce6a92c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8a99040e0, 0xc8ce6a92c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff110, 0xc802c3e000, 0xb1b, 0xc00, 0x347cc2bd, 0x280bcc0, 0xc802c3e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 416 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8ace1c0bdb77b699)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc826e1d080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc826e1d080, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc826e1d080, 0x28006b0, 0xc7384277a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6eb4c98a0, 0xc826e1d080, 0xc827f8fa90, 0xc738427868)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6eb4c98a0, 0xc5c4c817d0, 0xc826e1d080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6eb4c98c0, 0xc826e1d080, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff260, 0xc8bbfbf800, 0x8ae, 0x900, 0x392a11e8, 0x280bcc0, 0xc8bbfbf800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 417 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfdda7eda1ea2c819)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f0c510a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f0c510a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f0c510a0, 0x28006b0, 0xc8320a5200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc574a76b20, 0xc6f0c510a0, 0xc604875a90, 0xc8320a52b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc574a76b20, 0xc5c4c81830, 0xc6f0c510a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc574a76b40, 0xc6f0c510a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff3b0, 0xc85fa09000, 0x807, 0x900, 0x38c6e349, 0x280bcc0, 0xc85fa09000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff3b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 418 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x24b1154436b76e75)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc462878da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc462878da0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc462878da0, 0x28006b0, 0xc8462ddb00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c8268b40, 0xc462878da0, 0xc5ab8d1a90, 0xc8462ddbb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c8268b40, 0xc5c4c81890, 0xc462878da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c8268b60, 0xc462878da0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff500, 0xc8bb17e000, 0x79f, 0x900, 0x2839c8b5, 0x280bcc0, 0xc8bb17e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 419 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3c12ca70f7476cd2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81f6ea720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81f6ea720, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81f6ea720, 0x28006b0, 0xc4f12e2240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7d43aeb60, 0xc81f6ea720, 0xc5d478da90, 0xc4f12e22f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7d43aeb60, 0xc5c4c818f0, 0xc81f6ea720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7d43aeb80, 0xc81f6ea720, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff650, 0xc7e1edc000, 0x5dd, 0x6a0, 0x33f09dc9, 0x280bcc0, 0xc7e1edc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 420 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8621d67945d00bc8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6cba65ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6cba65ee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6cba65ee0, 0x28006b0, 0xc424dac5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6b746d860, 0xc6cba65ee0, 0xc58414fa90, 0xc424dac658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6b746d860, 0xc5c4c81950, 0xc6cba65ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6b746d880, 0xc6cba65ee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff7a0, 0xc72e93a000, 0x9f5, 0xc00, 0x218d34d9, 0x280bcc0, 0xc72e93a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff7a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 421 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1474b1ad6e1b3230)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78b874980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78b874980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78b874980, 0x28006b0, 0xc6f5cc2240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc731b80780, 0xc78b874980, 0xc586a29a90, 0xc6f5cc22f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc731b80780, 0xc5c4c819b0, 0xc78b874980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc731b807c0, 0xc78b874980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ff8f0, 0xc7b9938a00, 0x536, 0x6a0, 0x39cc5377, 0x280bcc0, 0xc7b9938a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ff8f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 422 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x81aa40e8af8ea2e0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7d60bccc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7d60bccc0, 0x1a753c0, 0xc582263a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7d60bccc0, 0x28006b0, 0xc78db638c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81c9934c0, 0xc7d60bccc0, 0xc582263a90, 0xc78db63978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81c9934c0, 0xc5c4c81a10, 0xc7d60bccc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81c9934e0, 0xc7d60bccc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ffa40, 0xc7aa834800, 0x7b5, 0x900, 0x3a8a12bd, 0x280bcc0, 0xc7aa834800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ffa40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 423 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x524941f6d210dbc1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc828654120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc828654120, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc828654120, 0x28006b0, 0xc7f6297e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4aa078c60, 0xc828654120, 0xc5da8f9a90, 0xc7f6297f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4aa078c60, 0xc5c4c81a70, 0xc828654120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4aa078c80, 0xc828654120, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ffb90, 0xc7d7033000, 0x765, 0x900, 0x3a3e70c9, 0x280bcc0, 0xc7d7033000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ffb90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 424 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x855ea97e1aad7236)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e3601a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e3601a20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e3601a20, 0x28006b0, 0xc6a74a6900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc764515da0, 0xc4e3601a20, 0xc5e6d25a90, 0xc6a74a69b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc764515da0, 0xc5c4c81ad0, 0xc4e3601a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc764515dc0, 0xc4e3601a20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ffce0, 0xc7d61d6000, 0x9f7, 0xc00, 0x13f5949c, 0x280bcc0, 0xc7d61d6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ffce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 425 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1addcef5dc84fdb2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8838f77e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8838f77e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8838f77e0, 0x28006b0, 0xc88e1617a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80b32bd40, 0xc8838f77e0, 0xc5aeacba90, 0xc88e161858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80b32bd40, 0xc5c4c81b30, 0xc8838f77e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80b32bd60, 0xc8838f77e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56ffe30, 0xc8ca560000, 0x6e8, 0x900, 0x5134b2e, 0x280bcc0, 0xc8ca560000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56ffe30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 426 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x213d43a50e316c71)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc577ccec60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc577ccec60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc577ccec60, 0x28006b0, 0xc68ac73e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc62bec1440, 0xc577ccec60, 0xc5adcbba90, 0xc68ac73f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc62bec1440, 0xc5c4c81b90, 0xc577ccec60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc62bec1460, 0xc577ccec60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d56fff80, 0xc7f854b000, 0x7d1, 0x900, 0x13b25f93, 0x280bcc0, 0xc7f854b000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d56fff80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 427 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5110d6dc38a11d47)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6db801f20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6db801f20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6db801f20, 0x28006b0, 0xc537d3d200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ac3687c0, 0xc6db801f20, 0xc5da5cda90, 0xc537d3d2d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ac3687c0, 0xc5c4c81bf0, 0xc6db801f20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ac3687e0, 0xc6db801f20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57540e0, 0xc97bedb000, 0x889, 0x900, 0x9578ca9, 0x280bcc0, 0xc97bedb000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57540e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 428 [semacquire, 9419 minutes]:
sync.runtime_SemacquireMutex(0xc4206bb8c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bb8c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x73ccb3b61fd60d03)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74d66b6a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74d66b6a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74d66b6a0, 0x28006b0, 0xc80093b8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f0cc4ba0, 0xc74d66b6a0, 0xc57ec11a90, 0xc80093b978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f0cc4ba0, 0xc5c4c81c50, 0xc74d66b6a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f0cc4bc0, 0xc74d66b6a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754230, 0xc7bea5ee00, 0xc5, 0xc8, 0x151ae420, 0x280bcc0, 0xc7bea5ee00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 429 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa28f48ac5b1ba44e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e5307660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e5307660, 0x1a753c0, 0xc5d3587a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e5307660, 0x28006b0, 0xc84ca30d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70edc2ca0, 0xc6e5307660, 0xc5d3587a90, 0xc84ca30e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70edc2ca0, 0xc5c4c81cb0, 0xc6e5307660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70edc2cc0, 0xc6e5307660, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754380, 0xc7bf296800, 0x6e9, 0x900, 0x34ba0427, 0x280bcc0, 0xc7bf296800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 430 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8bac6b949a2b0998)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a0e6f4e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a0e6f4e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a0e6f4e0, 0x28006b0, 0xc6353b26c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc75e2204c0, 0xc7a0e6f4e0, 0xc5b3ef9a90, 0xc6353b2778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc75e2204c0, 0xc5c4c81d10, 0xc7a0e6f4e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc75e220500, 0xc7a0e6f4e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57544d0, 0xc864360000, 0x9a6, 0xc00, 0x108f2a12, 0x280bcc0, 0xc864360000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57544d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 431 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb41d5211cd4de52e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc487b92ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc487b92ea0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc487b92ea0, 0x28006b0, 0xc7ea0f4b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4ccc5b900, 0xc487b92ea0, 0xc5a2dafa90, 0xc7ea0f4bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4ccc5b900, 0xc5c4c81d70, 0xc487b92ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4ccc5b920, 0xc487b92ea0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754620, 0xc7f6fd7000, 0x8ea, 0x900, 0x377c6873, 0x280bcc0, 0xc7f6fd7000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 432 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x48ebaa0e0332d8d9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6a84a4400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6a84a4400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6a84a4400, 0x28006b0, 0xc6589e5d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86a0dd900, 0xc6a84a4400, 0xc63b783a90, 0xc6589e5df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86a0dd900, 0xc5c4c81dd0, 0xc6a84a4400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86a0dd940, 0xc6a84a4400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754770, 0xc846f54800, 0x7fc, 0x900, 0x23673184, 0x280bcc0, 0xc846f54800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 433 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc1977f9b65a80bd6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc474b34dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc474b34dc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc474b34dc0, 0x28006b0, 0xc5bd87d440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f0a36f00, 0xc474b34dc0, 0xc8abc7fa90, 0xc5bd87d4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f0a36f00, 0xc5c4c81e30, 0xc474b34dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f0a36f60, 0xc474b34dc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57548c0, 0xc848d51800, 0x756, 0x900, 0xe8c58c8, 0x280bcc0, 0xc848d51800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57548c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 434 [semacquire, 9302 minutes]:
sync.runtime_SemacquireMutex(0xc4206d4b04)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d4b00)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6aae7cc019f6129a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b2886800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b2886800, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b2886800, 0x28006b0, 0xc5d4342900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7a0697c00, 0xc8b2886800, 0xc5ff509a90, 0xc5d43429c8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7a0697c00, 0xc5c4c81e90, 0xc8b2886800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7a0697c40, 0xc8b2886800, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754a10, 0xc87e5b4a00, 0x590, 0x6a0, 0x645a809, 0x280bcc0, 0xc87e5b4a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754a10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 435 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd0d46c30b8469593)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e0c56b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e0c56b00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e0c56b00, 0x28006b0, 0xc7e57947e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc61d5f7000, 0xc6e0c56b00, 0xc63d8cfa90, 0xc7e5794898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc61d5f7000, 0xc5c4c81ef0, 0xc6e0c56b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc61d5f7020, 0xc6e0c56b00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754b60, 0xc79dfb2000, 0xc00, 0xc00, 0x34d42c50, 0x280bcc0, 0xc79dfb2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754b60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 436 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x19b0846136653dc7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56a4ec7c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56a4ec7c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56a4ec7c0, 0x28006b0, 0xc8ca05eb40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7a5e4a0e0, 0xc56a4ec7c0, 0xc5deac9a90, 0xc8ca05ebf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7a5e4a0e0, 0xc5c4c81f50, 0xc56a4ec7c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7a5e4a100, 0xc56a4ec7c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754cb0, 0xc857a96000, 0x7dd, 0x900, 0x29c9245c, 0x280bcc0, 0xc857a96000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754cb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 437 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb40e8c3e7ab96cf5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc80e23a5c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc80e23a5c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc80e23a5c0, 0x28006b0, 0xc69b6a4d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c5934300, 0xc80e23a5c0, 0xc5df285a90, 0xc69b6a4e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c5934300, 0xc5c4c81fb0, 0xc80e23a5c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c5934320, 0xc80e23a5c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754e00, 0xc7f754a000, 0x8e3, 0x900, 0x197c5eb8, 0x280bcc0, 0xc7f754a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754e00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 438 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd10185ae9222e669)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc689ae5580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc689ae5580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc689ae5580, 0x28006b0, 0xc7fa6c39e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d86cc160, 0xc689ae5580, 0xc5dda03a90, 0xc7fa6c3aa8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d86cc160, 0xc5c2e3a030, 0xc689ae5580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d86cc180, 0xc689ae5580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5754f50, 0xc7a26e5800, 0x806, 0x900, 0x2a54ad4, 0x280bcc0, 0xc7a26e5800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5754f50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 439 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x87aa6b618fa45ce2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4886aa400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4886aa400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4886aa400, 0x28006b0, 0xc796bf50e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b3169360, 0xc4886aa400, 0xc5d8893a90, 0xc796bf5198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b3169360, 0xc5c2e3a090, 0xc4886aa400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b31693a0, 0xc4886aa400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57550a0, 0xc97bf18000, 0x7ed, 0x900, 0x1d52aab0, 0x280bcc0, 0xc97bf18000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57550a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 440 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x30948a90e35a544c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc64febce80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc64febce80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc64febce80, 0x28006b0, 0xc58d165560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc55522a2c0, 0xc64febce80, 0xc5b3ef3a90, 0xc58d165618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc55522a2c0, 0xc5c2e3a0f0, 0xc64febce80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc55522a2e0, 0xc64febce80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755260, 0xc89195b800, 0x893, 0x900, 0x1b3ea9fc, 0x280bcc0, 0xc89195b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 441 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xde829c8fe489a7c5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc71b158ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc71b158ec0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc71b158ec0, 0x28006b0, 0xc7c42950e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc568e63c00, 0xc71b158ec0, 0xc5cc6dba90, 0xc7c4295198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc568e63c00, 0xc5c2e3a150, 0xc71b158ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc568e63c20, 0xc71b158ec0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57553b0, 0xc86a8d8000, 0xad2, 0xc00, 0x9c1dec0, 0x280bcc0, 0xc86a8d8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57553b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 442 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb6be3a114b9b1e49)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc840702980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc840702980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc840702980, 0x28006b0, 0xc524f14120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc610134660, 0xc840702980, 0xc5b86c7a90, 0xc524f141d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc610134660, 0xc5c2e3a1b0, 0xc840702980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc610134680, 0xc840702980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755500, 0xc8825e4000, 0x719, 0x900, 0x26c65ee, 0x280bcc0, 0xc8825e4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 443 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x362091a1aa2762e3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc836e01320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc836e01320, 0x1a753c0, 0xc5e2d7da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc836e01320, 0x28006b0, 0xc744ae2900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8315ace60, 0xc836e01320, 0xc5e2d7da90, 0xc744ae29b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8315ace60, 0xc5c2e3a210, 0xc836e01320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8315ace80, 0xc836e01320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755650, 0xc987d38000, 0x829, 0x900, 0x163d49c3, 0x280bcc0, 0xc987d38000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 444 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe8014fcab1c4276b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73e9e99e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73e9e99e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73e9e99e0, 0x28006b0, 0xc7dee64c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc635f5a3c0, 0xc73e9e99e0, 0xc605c55a90, 0xc7dee64d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc635f5a3c0, 0xc5c2e3a270, 0xc73e9e99e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc635f5a3e0, 0xc73e9e99e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57557a0, 0xc984068000, 0x92c, 0xc00, 0x470c787, 0x280bcc0, 0xc984068000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57557a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 445 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xaab10688275382e1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e44268c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e44268c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e44268c0, 0x28006b0, 0xc8c93ffc20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6a2538400, 0xc4e44268c0, 0xc631459a90, 0xc8c93ffcd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6a2538400, 0xc5c2e3a2d0, 0xc4e44268c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6a2538420, 0xc4e44268c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d57558f0, 0xc98bace000, 0xa56, 0xc00, 0x41d8f53, 0x280bcc0, 0xc98bace000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d57558f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 446 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe2ed972e8f35e42)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc79315afc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc79315afc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc79315afc0, 0x28006b0, 0xc726000a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6c81d05a0, 0xc79315afc0, 0xc602a03a90, 0xc726000ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6c81d05a0, 0xc5c2e3a330, 0xc79315afc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6c81d05c0, 0xc79315afc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755a40, 0xc7abab7000, 0x763, 0x900, 0x91af5db, 0x280bcc0, 0xc7abab7000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755a40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 447 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9297208564dc0c38)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc685b1c200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc685b1c200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc685b1c200, 0x28006b0, 0xc8854886c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc72771b920, 0xc685b1c200, 0xc5ac723a90, 0xc885488778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc72771b920, 0xc5c2e3a390, 0xc685b1c200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc72771b9a0, 0xc685b1c200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755b90, 0xc86f842800, 0x8a4, 0x900, 0xa14ee2e, 0x280bcc0, 0xc86f842800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755b90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 448 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x191b41a4e1bcdf8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc808dccd60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc808dccd60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc808dccd60, 0x28006b0, 0xc829543d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8623fa2c0, 0xc808dccd60, 0xc5db22ba90, 0xc829543df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8623fa2c0, 0xc5c2e3a3f0, 0xc808dccd60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8623fa2e0, 0xc808dccd60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755ce0, 0xc8cd220000, 0x725, 0x900, 0x20f360c8, 0x280bcc0, 0xc8cd220000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755ce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 449 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x80ee01eda6e05437)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74d73b9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74d73b9a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74d73b9a0, 0x28006b0, 0xc7dd35a7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc741b402a0, 0xc74d73b9a0, 0xc5c31eda90, 0xc7dd35a8a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc741b402a0, 0xc5c2e3a450, 0xc74d73b9a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc741b402c0, 0xc74d73b9a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755e30, 0xc892bc6000, 0x8ee, 0x900, 0x16717408, 0x280bcc0, 0xc892bc6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755e30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 450 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x92ad8392d97a0289)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88a261100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88a261100, 0x1a753c0, 0xc7fb09b200)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88a261100, 0x28006b0, 0xc7fb09b200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ea893720, 0xc88a261100, 0xc6385a7a90, 0xc7fb09b2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ea893720, 0xc5c2e3a4b0, 0xc88a261100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7ea893760, 0xc88a261100, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d5755f80, 0xc8b75d1800, 0x7ef, 0x900, 0x36ec6bbd, 0x280bcc0, 0xc8b75d1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d5755f80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 451 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x508be991f6901664)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc54f9dfae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc54f9dfae0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc54f9dfae0, 0x28006b0, 0xc86ed2dd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d9fd8680, 0xc54f9dfae0, 0xc60461da90, 0xc86ed2ddf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d9fd8680, 0xc5c2e3a510, 0xc54f9dfae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d9fd86a0, 0xc54f9dfae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce0e0, 0xc7f677e800, 0x831, 0x900, 0x2328cc5a, 0x280bcc0, 0xc7f677e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 452 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2c7ef09d1af7930f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7c62fea80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7c62fea80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7c62fea80, 0x28006b0, 0xc751d4b440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4edbeb320, 0xc7c62fea80, 0xc586acfa90, 0xc751d4b508)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4edbeb320, 0xc5c2e3a570, 0xc7c62fea80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4edbeb340, 0xc7c62fea80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce230, 0xc7d6c55500, 0x623, 0x6a0, 0x19ae86da, 0x280bcc0, 0xc7d6c55500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 453 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1fb44af0d9779262)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a1710400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a1710400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a1710400, 0x28006b0, 0xc5d41aeea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6376691c0, 0xc7a1710400, 0xc587f57a90, 0xc5d41aef58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6376691c0, 0xc5c2e3a5d0, 0xc7a1710400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6376691e0, 0xc7a1710400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce380, 0xc7d4ebd800, 0x827, 0x900, 0x1aab3e5, 0x280bcc0, 0xc7d4ebd800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 454 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb3cea9a67e458a65)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4bdbaf3a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4bdbaf3a0, 0x1a753c0, 0xc5d75c7a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4bdbaf3a0, 0x28006b0, 0xc521e96360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7501660e0, 0xc4bdbaf3a0, 0xc5d75c7a90, 0xc521e96418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7501660e0, 0xc5c2e3a630, 0xc4bdbaf3a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc750166100, 0xc4bdbaf3a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce4d0, 0xc7bf29f800, 0x735, 0x900, 0x9781429, 0x280bcc0, 0xc7bf29f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce4d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 455 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x587429199344959e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc70a71b840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc70a71b840, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc70a71b840, 0x28006b0, 0xc843dbea20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc702dcf000, 0xc70a71b840, 0xc5d8151a90, 0xc843dbead8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc702dcf000, 0xc5c2e3a690, 0xc70a71b840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc702dcf020, 0xc70a71b840, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce620, 0xc821555000, 0x79b, 0x900, 0xfa7d873, 0x280bcc0, 0xc821555000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 456 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf75cdf3d710be1f5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74e0e1b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74e0e1b00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74e0e1b00, 0x28006b0, 0xc751994fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc807b1ba60, 0xc74e0e1b00, 0xc5d5ebda90, 0xc751995088)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc807b1ba60, 0xc5c2e3a6f0, 0xc74e0e1b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc807b1baa0, 0xc74e0e1b00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce770, 0xc88db9f500, 0x668, 0x6a0, 0xd924c1, 0x280bcc0, 0xc88db9f500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 457 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc3fbafdf088a4896)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7acc2a2a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7acc2a2a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7acc2a2a0, 0x28006b0, 0xc8087a87e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54d1a8520, 0xc7acc2a2a0, 0xc601feda90, 0xc8087a8898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54d1a8520, 0xc5c2e3a750, 0xc7acc2a2a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54d1a8540, 0xc7acc2a2a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ce8c0, 0xc893670800, 0x836, 0x900, 0x23d768f, 0x280bcc0, 0xc893670800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ce8c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 458 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x59440a61b2a250e4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7aeeb1d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7aeeb1d20, 0x1a753c0, 0xc5ae32da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7aeeb1d20, 0x28006b0, 0xc50bb1c900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8603ff060, 0xc7aeeb1d20, 0xc5ae32da90, 0xc50bb1c9b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8603ff060, 0xc5c2e3a7b0, 0xc7aeeb1d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8603ff080, 0xc7aeeb1d20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cea10, 0xc7bf29b000, 0x74b, 0x900, 0x128272cf, 0x280bcc0, 0xc7bf29b000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cea10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 459 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x23c32c2fe14bf543)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc83d1e2800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc83d1e2800, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc83d1e2800, 0x28006b0, 0xc7b1ba4240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc60f1ca580, 0xc83d1e2800, 0xc5b076fa90, 0xc7b1ba42f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc60f1ca580, 0xc5c2e3a810, 0xc83d1e2800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc60f1ca5a0, 0xc83d1e2800, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03ceb60, 0xc7d81b8000, 0x901, 0xc00, 0xe58842, 0x280bcc0, 0xc7d81b8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03ceb60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 460 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x806d5339a3da5a91)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc689ec33c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc689ec33c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc689ec33c0, 0x28006b0, 0xc7edf13320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5588b1640, 0xc689ec33c0, 0xc5c02fda90, 0xc7edf133d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5588b1640, 0xc5c2e3a870, 0xc689ec33c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5588b1660, 0xc689ec33c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cecb0, 0xc892c4e000, 0x9d2, 0xc00, 0x5c902be, 0x280bcc0, 0xc892c4e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cecb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 461 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd34bb082c9fe7899)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc514e1fd40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc514e1fd40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc514e1fd40, 0x28006b0, 0xc805e9ea20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc516dcacc0, 0xc514e1fd40, 0xc5de535a90, 0xc805e9ead8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc516dcacc0, 0xc5c2e3a8d0, 0xc514e1fd40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc516dcad00, 0xc514e1fd40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cee00, 0xc88feb1000, 0x76c, 0x900, 0x33ea2c30, 0x280bcc0, 0xc88feb1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cee00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 462 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9057270663dbedb8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7da148b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7da148b40, 0x1a753c0, 0xc602edda18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7da148b40, 0x28006b0, 0xc5f9485680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8b09eef60, 0xc7da148b40, 0xc602edda90, 0xc5f9485738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8b09eef60, 0xc5c2e3a930, 0xc7da148b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8b09eef80, 0xc7da148b40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cef50, 0xc987aa5800, 0x6a8, 0x900, 0x3a3d5ae5, 0x280bcc0, 0xc987aa5800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cef50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 463 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc3d3530aaa639a53)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc703624700, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc703624700, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc703624700, 0x28006b0, 0xc60a330360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7aec77d00, 0xc703624700, 0xc63f1d9a90, 0xc60a330418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7aec77d00, 0xc5c2e3a990, 0xc703624700, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7aec77d20, 0xc703624700, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf0a0, 0xc88ec1f000, 0x7a7, 0x900, 0x22483544, 0x280bcc0, 0xc88ec1f000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf0a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 464 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x80359707d088c66a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6414bc640, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6414bc640, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6414bc640, 0x28006b0, 0xc886f15200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8ba188360, 0xc6414bc640, 0xc583193a90, 0xc886f152b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8ba188360, 0xc5c2e3a9f0, 0xc6414bc640, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8ba188380, 0xc6414bc640, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf1f0, 0xc8c9a3a000, 0x9d1, 0xc00, 0x3354b1b6, 0x280bcc0, 0xc8c9a3a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf1f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 465 [semacquire, 10606 minutes]:
sync.runtime_SemacquireMutex(0xc4206e9984)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206e9980)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfc88767bb6902c0d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc52151a1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc52151a1c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc52151a1c0, 0x28006b0, 0xc6769c4360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86f384880, 0xc52151a1c0, 0xc5dc6c5a90, 0xc6769c4428)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86f384880, 0xc5c2e3aa50, 0xc52151a1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86f3848a0, 0xc52151a1c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf340, 0xc8cc359500, 0x48a, 0x6a0, 0x382a36e5, 0x280bcc0, 0xc8cc359500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 466 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa9bd0c264512cb4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86bdaa080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86bdaa080, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86bdaa080, 0x28006b0, 0xc8cbad50e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc816253560, 0xc86bdaa080, 0xc5d3e6fa90, 0xc8cbad5198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc816253560, 0xc5c2e3aab0, 0xc86bdaa080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc816253580, 0xc86bdaa080, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf490, 0xc7cf0aa000, 0x9ee, 0xc00, 0x26bb76cf, 0x280bcc0, 0xc7cf0aa000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 467 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x84a9d99e44fcd503)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73e44cb20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73e44cb20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73e44cb20, 0x28006b0, 0xc591bfc360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7cf47e960, 0xc73e44cb20, 0xc5d9583a90, 0xc591bfc428)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7cf47e960, 0xc5c2e3ab10, 0xc73e44cb20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7cf47e980, 0xc73e44cb20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf5e0, 0xc986e7c000, 0x9e7, 0xc00, 0x2b4a4c2f, 0x280bcc0, 0xc986e7c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf5e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 468 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfd1fcb7ccb856898)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7138ca720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7138ca720, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7138ca720, 0x28006b0, 0xc553544900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc833923320, 0xc7138ca720, 0xc587e01a90, 0xc5535449b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc833923320, 0xc5c2e3ab70, 0xc7138ca720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc833923340, 0xc7138ca720, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf730, 0xc85fe3a000, 0x94f, 0xc00, 0xcb77232, 0x280bcc0, 0xc85fe3a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf730, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 469 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x667fe46c79f95eaf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4ddfd54e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4ddfd54e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4ddfd54e0, 0x28006b0, 0xc841fcc360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc64a2cae80, 0xc4ddfd54e0, 0xc603b6fa90, 0xc841fcc418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc64a2cae80, 0xc5c2e3abd0, 0xc4ddfd54e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc64a2caec0, 0xc4ddfd54e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cf880, 0xc893679800, 0x821, 0x900, 0x35c803a4, 0x280bcc0, 0xc893679800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cf880, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 470 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa309dd29ee46cb48)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88d778200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88d778200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88d778200, 0x28006b0, 0xc81bbbf440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc832a1ffa0, 0xc88d778200, 0xc57fc09a90, 0xc81bbbf4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc832a1ffa0, 0xc5c2e3ac30, 0xc88d778200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc832a1ffc0, 0xc88d778200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cfa40, 0xc7d6c78000, 0xb38, 0xc00, 0x2c9e10d9, 0x280bcc0, 0xc7d6c78000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cfa40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 471 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4a128e381c2eb3f3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc551bb3a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc551bb3a20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc551bb3a20, 0x28006b0, 0xc58c922a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc56a6697c0, 0xc551bb3a20, 0xc5e255da90, 0xc58c922ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc56a6697c0, 0xc5c2e3ac90, 0xc551bb3a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc56a669820, 0xc551bb3a20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cfb90, 0xc857aa3800, 0x7ed, 0x900, 0xa842b87, 0x280bcc0, 0xc857aa3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cfb90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 472 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x33cbe4d1faf78647)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5a0365b60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5a0365b60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5a0365b60, 0x28006b0, 0xc7343ae5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc51ae8af20, 0xc5a0365b60, 0xc5a3e9ba90, 0xc7343ae668)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc51ae8af20, 0xc5c2e3acf0, 0xc5a0365b60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc51ae8b140, 0xc5a0365b60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cfce0, 0xc981b28a00, 0x675, 0x6a0, 0x2beaf76c, 0x280bcc0, 0xc981b28a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cfce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 473 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x59cfa1be6287e4c0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88f7d68c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88f7d68c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88f7d68c0, 0x28006b0, 0xc5d42ea480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc881542440, 0xc88f7d68c0, 0xc5c02ffa90, 0xc5d42ea538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc881542440, 0xc5c2e3ad50, 0xc88f7d68c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc881542460, 0xc88f7d68c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cfe30, 0xc8baf75800, 0x7de, 0x900, 0x2a2d0c50, 0x280bcc0, 0xc8baf75800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cfe30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 474 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x19baee1ca09d9f64)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82b93a240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82b93a240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82b93a240, 0x28006b0, 0xc5728106c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7a5180380, 0xc82b93a240, 0xc63b26ba90, 0xc572810778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7a5180380, 0xc5c2e3adb0, 0xc82b93a240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7a51803a0, 0xc82b93a240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e03cff80, 0xc8b2a02000, 0xa11, 0xc00, 0x127bcd99, 0x280bcc0, 0xc8b2a02000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e03cff80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 475 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2bdd93510bed02ce)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6bcb950e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6bcb950e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6bcb950e0, 0x28006b0, 0xc62889dc20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc781a34180, 0xc6bcb950e0, 0xc5b51cda90, 0xc62889dcd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc781a34180, 0xc5c2e3ae10, 0xc6bcb950e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc781a341a0, 0xc6bcb950e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10700e0, 0xc7d3abe000, 0x8c7, 0x900, 0x1dad763, 0x280bcc0, 0xc7d3abe000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10700e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 476 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6e1ab1b724fcc4ae)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5575f0b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5575f0b80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5575f0b80, 0x28006b0, 0xc50bf3f9e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc660747b40, 0xc5575f0b80, 0xc57de67a90, 0xc50bf3fa98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc660747b40, 0xc5c2e3ae70, 0xc5575f0b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc660747ba0, 0xc5575f0b80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070230, 0xc982c6e000, 0x7e2, 0x900, 0x38b173f6, 0x280bcc0, 0xc982c6e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 477 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xbb40fcc65b23d600)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc58eabd3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc58eabd3e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc58eabd3e0, 0x28006b0, 0xc8150ee000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc69c7e67c0, 0xc58eabd3e0, 0xc58452fa90, 0xc8150ee0b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc69c7e67c0, 0xc5c2e3aed0, 0xc58eabd3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc69c7e67e0, 0xc58eabd3e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070380, 0xc7b1e84000, 0xc11, 0x1000, 0x115cb20d, 0x280bcc0, 0xc7b1e84000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 478 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2898770ad28a5ff2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc67b50c060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc67b50c060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc67b50c060, 0x28006b0, 0xc53f182d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc674a2fb40, 0xc67b50c060, 0xc57eb39a90, 0xc53f182e48)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc674a2fb40, 0xc5c2e3af30, 0xc67b50c060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc674a2fb60, 0xc67b50c060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10704d0, 0xc82024f800, 0x7b6, 0x900, 0x37dd8b43, 0x280bcc0, 0xc82024f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10704d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 479 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe4dd0cdde6529ca0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73d072120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73d072120, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73d072120, 0x28006b0, 0xc9838a47e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4b263c780, 0xc73d072120, 0xc580137a90, 0xc9838a4898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4b263c780, 0xc5c2e3af90, 0xc73d072120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4b263c7c0, 0xc73d072120, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070690, 0xc858491800, 0x755, 0x900, 0x263da01f, 0x280bcc0, 0xc858491800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070690, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 480 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x76ee7fa991c9fd5a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc62c2d3040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc62c2d3040, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc62c2d3040, 0x28006b0, 0xc84c011e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc88e3e27e0, 0xc62c2d3040, 0xc5bfd55a90, 0xc84c011f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc88e3e27e0, 0xc5c2e3aff0, 0xc62c2d3040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc88e3e2800, 0xc62c2d3040, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10707e0, 0xc869b0e000, 0xa13, 0xc00, 0x15fd347b, 0x280bcc0, 0xc869b0e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10707e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 481 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x834417de3b62613a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88cc95e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88cc95e60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88cc95e60, 0x28006b0, 0xc64a472ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86f253200, 0xc88cc95e60, 0xc5b681ba90, 0xc64a472f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86f253200, 0xc5c2e3b050, 0xc88cc95e60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86f253220, 0xc88cc95e60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070930, 0xc984f12800, 0x8d4, 0x900, 0x34c4e20b, 0x280bcc0, 0xc984f12800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070930, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 482 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xbd96f5d3b3eb37dd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc867fcfa60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc867fcfa60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc867fcfa60, 0x28006b0, 0xc823cc2900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ea883720, 0xc867fcfa60, 0xc596c25a90, 0xc823cc29b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ea883720, 0xc5c2e3b0b0, 0xc867fcfa60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ea883740, 0xc867fcfa60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070a80, 0xc8bafec800, 0x77e, 0x900, 0xc01b9ff, 0x280bcc0, 0xc8bafec800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070a80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 483 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfcc55fe84e5b9289)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6a3977de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6a3977de0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6a3977de0, 0x28006b0, 0xc8c7d33c20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc738a60e80, 0xc6a3977de0, 0xc5b871da90, 0xc8c7d33cd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc738a60e80, 0xc5c2e3b110, 0xc6a3977de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc738a60ea0, 0xc6a3977de0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070bd0, 0xc890d2c800, 0x8d4, 0x900, 0x2f8d1b15, 0x280bcc0, 0xc890d2c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070bd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 484 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcfbb07a348235206)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73c97d8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73c97d8c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73c97d8c0, 0x28006b0, 0xc810cefe60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc55c7ab980, 0xc73c97d8c0, 0xc5d5eb9a90, 0xc810ceff18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc55c7ab980, 0xc5c2e3b170, 0xc73c97d8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc55c7ab9c0, 0xc73c97d8c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070d20, 0xc7ad2e7500, 0x5e0, 0x6a0, 0xa968be3, 0x280bcc0, 0xc7ad2e7500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070d20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 485 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xab1f61a528b808f8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7787bfa40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7787bfa40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7787bfa40, 0x28006b0, 0xc7b0d7f200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc64ffde320, 0xc7787bfa40, 0xc58c3e9a90, 0xc7b0d7f2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc64ffde320, 0xc5c2e3b1d0, 0xc7787bfa40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc64ffde360, 0xc7787bfa40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070e70, 0xc986ace000, 0x876, 0x900, 0x10c657f8, 0x280bcc0, 0xc986ace000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070e70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 486 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x862e52b5992fcc63)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc807a02580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc807a02580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc807a02580, 0x28006b0, 0xc7dcdd8ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e8451440, 0xc807a02580, 0xc5af7e9a90, 0xc7dcdd8f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e8451440, 0xc5c2e3b230, 0xc807a02580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e8451460, 0xc807a02580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1070fc0, 0xc85850a800, 0x84d, 0x900, 0x2cdd275f, 0x280bcc0, 0xc85850a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1070fc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 487 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xacafb8b5ffcb9f1f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8383668a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8383668a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8383668a0, 0x28006b0, 0xc878476ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc78e1a1640, 0xc8383668a0, 0xc5bfd71a90, 0xc878476f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc78e1a1640, 0xc5c2e3b290, 0xc8383668a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc78e1a1660, 0xc8383668a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071110, 0xc848086000, 0x810, 0x900, 0x137d69d2, 0x280bcc0, 0xc848086000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 488 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1ad0186662edf8f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8afad68a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8afad68a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8afad68a0, 0x28006b0, 0xc43633de60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4ac179600, 0xc8afad68a0, 0xc5d4a2ba90, 0xc43633df18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4ac179600, 0xc5c2e3b2f0, 0xc8afad68a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4ac179620, 0xc8afad68a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071260, 0xc8c6fb0000, 0x60a, 0x6a0, 0x179ccd76, 0x280bcc0, 0xc8c6fb0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 489 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5a45c60944446177)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc774e5e7c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc774e5e7c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc774e5e7c0, 0x28006b0, 0xc82b88c360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc55560e9a0, 0xc774e5e7c0, 0xc606fd7a90, 0xc82b88c418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc55560e9a0, 0xc5c2e3b350, 0xc774e5e7c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc55560e9c0, 0xc774e5e7c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10713b0, 0xc8a861a800, 0x873, 0x900, 0x365fcb72, 0x280bcc0, 0xc8a861a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10713b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 490 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd185b1e3afe78476)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7d0e9c5e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7d0e9c5e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7d0e9c5e0, 0x28006b0, 0xc7d7f4cd80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc793a589e0, 0xc7d0e9c5e0, 0xc6036bda90, 0xc7d7f4ce38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc793a589e0, 0xc5c2e3b3b0, 0xc7d0e9c5e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc793a58a00, 0xc7d0e9c5e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071500, 0xc8b6f87000, 0x7ef, 0x900, 0xd7679e6, 0x280bcc0, 0xc8b6f87000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 491 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x427536023c178370)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b74fee20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b74fee20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b74fee20, 0x28006b0, 0xc74b0a9c20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83fdc6b80, 0xc8b74fee20, 0xc6036c1a90, 0xc74b0a9cd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83fdc6b80, 0xc5c2e3b410, 0xc8b74fee20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83fdc6ba0, 0xc8b74fee20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071650, 0xc59382e000, 0x9a3, 0xc00, 0x2e5ef00f, 0x280bcc0, 0xc59382e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 492 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf1a1ace6cc23973c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5eef91240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5eef91240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5eef91240, 0x28006b0, 0xc7e62d07e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7aa9cd860, 0xc5eef91240, 0xc5aff65a90, 0xc7e62d0898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7aa9cd860, 0xc5c2e3b470, 0xc5eef91240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7aa9cd8a0, 0xc5eef91240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10717a0, 0xc81e7a1800, 0x789, 0x900, 0x277067df, 0x280bcc0, 0xc81e7a1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10717a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 493 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa66553d275ee3e5d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc57e400120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc57e400120, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc57e400120, 0x28006b0, 0xc44ed23b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc522ec65c0, 0xc57e400120, 0xc5e38a1a90, 0xc44ed23bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc522ec65c0, 0xc5c2e3b4d0, 0xc57e400120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc522ec6640, 0xc57e400120, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e10718f0, 0xc7d75a6000, 0x928, 0xc00, 0x1748de4e, 0x280bcc0, 0xc7d75a6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e10718f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 494 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8a371777d1ea795e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b2196320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b2196320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b2196320, 0x28006b0, 0xc7625f5d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84359bf60, 0xc8b2196320, 0xc5a8afda90, 0xc7625f5df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84359bf60, 0xc5c2e3b530, 0xc8b2196320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84359bfa0, 0xc8b2196320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071a40, 0xc8c4c61800, 0x78f, 0x900, 0x251981e, 0x280bcc0, 0xc8c4c61800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071a40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 495 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe1fea2a32c446f02)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8707d80a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8707d80a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8707d80a0, 0x28006b0, 0xc736c726c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc55d740de0, 0xc8707d80a0, 0xc5aad8fa90, 0xc736c72778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc55d740de0, 0xc5c2e3b590, 0xc8707d80a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc55d740e00, 0xc8707d80a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071b90, 0xc7d6201000, 0x7bd, 0x900, 0x2c6d291c, 0x280bcc0, 0xc7d6201000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071b90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 496 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4caace2e49ab8fa2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc708f97d60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc708f97d60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc708f97d60, 0x28006b0, 0xc66a500900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f9e17300, 0xc708f97d60, 0xc5df36da90, 0xc66a5009b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f9e17300, 0xc5c2e3b5f0, 0xc708f97d60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f9e17320, 0xc708f97d60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071ce0, 0xc7d114f500, 0x59a, 0x6a0, 0x23605b44, 0x280bcc0, 0xc7d114f500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071ce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 497 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x574a7e63289a6e98)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4dbdbaba0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4dbdbaba0, 0x1a753c0, 0xc5d82e1d00)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4dbdbaba0, 0x28006b0, 0xc8ad948120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc561bc0540, 0xc4dbdbaba0, 0xc88c1b7a90, 0xc8ad9481d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc561bc0540, 0xc5c2e3b650, 0xc4dbdbaba0, 0xc561bc0580, 0x1a00660)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc561bc0580, 0xc4dbdbaba0, 0x27535e0, 0xc561bc0580)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071e30, 0xc980dda000, 0x92b, 0xc00, 0x23081c9, 0x280bcc0, 0xc980dda000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071e30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 498 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc6a974ef5126f84f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc824807780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc824807780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc824807780, 0x28006b0, 0xc72b8466c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86669a620, 0xc824807780, 0xc5dd381a90, 0xc72b846778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86669a620, 0xc5c2e3b6b0, 0xc824807780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86669a660, 0xc824807780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e1071f80, 0xc83819f800, 0x875, 0x900, 0x272ec242, 0x280bcc0, 0xc83819f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e1071f80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 499 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5c89c31b362128bc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e1d28be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e1d28be0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e1d28be0, 0x28006b0, 0xc8455087e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc65502f980, 0xc4e1d28be0, 0xc604ef3a90, 0xc845508898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc65502f980, 0xc5c2e3b710, 0xc4e1d28be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc65502f9a0, 0xc4e1d28be0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4150, 0xc85715a000, 0x91d, 0xc00, 0x2b3acd6a, 0x280bcc0, 0xc85715a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4150, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 500 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb3340411587003a2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e5a766c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e5a766c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e5a766c0, 0x28006b0, 0xc59f6eb8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5eae3eb00, 0xc7e5a766c0, 0xc598d1ba90, 0xc59f6eb978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5eae3eb00, 0xc5c2e3b770, 0xc7e5a766c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5eae3eb20, 0xc7e5a766c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d42a0, 0xc7adf0c000, 0xa46, 0xc00, 0x295da035, 0x280bcc0, 0xc7adf0c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d42a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 501 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3eb9b8bdac3e737a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc462b25fa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc462b25fa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc462b25fa0, 0x28006b0, 0xc7c56897a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54d94e8c0, 0xc462b25fa0, 0xc5a4a5fa90, 0xc7c5689858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54d94e8c0, 0xc5c2e3b7d0, 0xc462b25fa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54d94e8e0, 0xc462b25fa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d43f0, 0xc824a2a000, 0xa36, 0xc00, 0x1b3c1ea5, 0x280bcc0, 0xc824a2a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d43f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 502 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcb6629b8da40fc3b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82f696f00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82f696f00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82f696f00, 0x28006b0, 0xc66c22eea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ba5d6160, 0xc82f696f00, 0xc6036bba90, 0xc66c22ef58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ba5d6160, 0xc5c2e3b830, 0xc82f696f00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ba5d6180, 0xc82f696f00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4540, 0xc88e910000, 0x706, 0x900, 0x27328b6c, 0x280bcc0, 0xc88e910000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4540, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 503 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6e5b7f56a7ba21)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc868167aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc868167aa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc868167aa0, 0x28006b0, 0xc81e412900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc722634360, 0xc868167aa0, 0xc605fd3a90, 0xc81e4129b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc722634360, 0xc5c2e3b890, 0xc868167aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc722634380, 0xc868167aa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4690, 0xc887724000, 0x953, 0xc00, 0x2d4ea30e, 0x280bcc0, 0xc887724000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4690, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 504 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x74fd620ab6e02ccd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4ae6f3000, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4ae6f3000, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4ae6f3000, 0x28006b0, 0xc65a4b8360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b1333ea0, 0xc4ae6f3000, 0xc5abcffa90, 0xc65a4b8418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b1333ea0, 0xc5c2e3b8f0, 0xc4ae6f3000, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b1333ec0, 0xc4ae6f3000, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d47e0, 0xc861600000, 0x5ac, 0x6a0, 0x622dbdd, 0x280bcc0, 0xc861600000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d47e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 505 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3cdb39419817c937)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5f236afe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5f236afe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5f236afe0, 0x28006b0, 0xc7e5176120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86a664b00, 0xc5f236afe0, 0xc5e12e7a90, 0xc7e51761d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86a664b00, 0xc5c2e3b950, 0xc5f236afe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86a664b40, 0xc5f236afe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4930, 0xc8b543e800, 0x8c5, 0x900, 0x16dc7c90, 0x280bcc0, 0xc8b543e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4930, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 506 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6529a60029cdecf7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc71fc881a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc71fc881a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc71fc881a0, 0x28006b0, 0xc79fcdf320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc73ed111e0, 0xc71fc881a0, 0xc6373fba90, 0xc79fcdf3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc73ed111e0, 0xc5c2e3b9b0, 0xc71fc881a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc73ed11220, 0xc71fc881a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4a80, 0xc82234f500, 0x577, 0x6a0, 0x28cc67b3, 0x280bcc0, 0xc82234f500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4a80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 507 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa526bc230a48f551)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5f0ee43e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5f0ee43e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5f0ee43e0, 0x28006b0, 0xc8b5dd5b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc775aa7260, 0xc5f0ee43e0, 0xc63b72fa90, 0xc8b5dd5bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc775aa7260, 0xc5c2e3ba10, 0xc5f0ee43e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc775aa72a0, 0xc5f0ee43e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4bd0, 0xc981b62000, 0x7e3, 0x900, 0x3399a2ed, 0x280bcc0, 0xc981b62000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4bd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 508 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xeea68fdb5ffb11c5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5e60e8ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5e60e8ee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5e60e8ee0, 0x28006b0, 0xc728527e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc69fcf62e0, 0xc5e60e8ee0, 0xc5bea5fa90, 0xc728527f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc69fcf62e0, 0xc5c2e3ba70, 0xc5e60e8ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc69fcf6300, 0xc5e60e8ee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4d20, 0xc7d2eee000, 0x782, 0x900, 0xe7f6ae2, 0x280bcc0, 0xc7d2eee000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4d20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 509 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdecfd90710f3e1d3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81c0fa200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81c0fa200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81c0fa200, 0x28006b0, 0xc64a2c0360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7cfb0ab20, 0xc81c0fa200, 0xc5ab8cda90, 0xc64a2c0418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7cfb0ab20, 0xc5c2e3bad0, 0xc81c0fa200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7cfb0ab40, 0xc81c0fa200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4e70, 0xc800e2e000, 0x944, 0xc00, 0x39574fa0, 0x280bcc0, 0xc800e2e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4e70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 510 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7c71028a2060b30f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a5c3ca60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a5c3ca60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a5c3ca60, 0x28006b0, 0xc4df1e47e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc82b2bdca0, 0xc7a5c3ca60, 0xc59867ba90, 0xc4df1e4898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc82b2bdca0, 0xc5c2e3bb30, 0xc7a5c3ca60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc82b2bdd00, 0xc7a5c3ca60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d4fc0, 0xc8c5928000, 0x778, 0x900, 0x52c8355, 0x280bcc0, 0xc8c5928000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d4fc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 511 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe4e1ad3cc52137de)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7ad01bc80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7ad01bc80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7ad01bc80, 0x28006b0, 0xc55a2bdd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6935ae480, 0xc7ad01bc80, 0xc5e40e7a90, 0xc55a2bddf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6935ae480, 0xc5c2e3bb90, 0xc7ad01bc80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6935ae4a0, 0xc7ad01bc80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5110, 0xc97fbec000, 0x954, 0xc00, 0x1efad07a, 0x280bcc0, 0xc97fbec000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 512 [select]:
net/http.(*Transport).getConn(0xc5e137e000, 0xc8185f09c0, 0x0, 0xc5e150c300, 0x4, 0xc83df66090, 0x2f, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:913 +0xb92
net/http.(*Transport).RoundTrip(0xc5e137e000, 0xc779fb8000, 0xc5e137e000, 0x0, 0x7fb000000000)
/usr/local/go/src/net/http/transport.go:378 +0x58c
net/http.send(0xc779fb8000, 0x274c2e0, 0xc5e137e000, 0x0, 0x0, 0x0, 0x0, 0x8, 0xc5ec9c8ff8, 0x40fe28)
/usr/local/go/src/net/http/client.go:249 +0x162
net/http.(*Client).send(0xc5c4c800f0, 0xc779fb8000, 0x0, 0x0, 0x0, 0xc5ec9c8ff8, 0x0, 0x1, 0xc5d5522211)
/usr/local/go/src/net/http/client.go:173 +0x108
net/http.(*Client).Do(0xc5c4c800f0, 0xc779fb8000, 0xc49a8981e0, 0xc779fb8000, 0xc49a8981e0)
/usr/local/go/src/net/http/client.go:595 +0x254
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x7fb0d6245ab8, 0xc49a8981e0, 0xc5c4c800f0, 0xc8af9fcf00, 0x9, 0xc5d5522270, 0x9)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x92
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0xc552baf440, 0x7fb0d6245ab8, 0xc49a8981e0, 0xed0d3a054, 0x10148474, 0x280bcc0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:312 +0x2c0
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:427 +0x42e
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 513 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8fe0219251f3f800)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc61c2eb400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc61c2eb400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc61c2eb400, 0x28006b0, 0xc8814c8c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ccfb4e40, 0xc61c2eb400, 0xc57fd71a90, 0xc8814c8d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ccfb4e40, 0xc5c2e3bc50, 0xc61c2eb400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ccfb4e60, 0xc61c2eb400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d53b0, 0xc97fc30000, 0x881, 0x900, 0x26eeb1d8, 0x280bcc0, 0xc97fc30000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d53b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 514 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe4f7cfa8d980d512)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4273bebe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4273bebe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4273bebe0, 0x28006b0, 0xc5642d3680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87eb04ac0, 0xc4273bebe0, 0xc5d7d1ba90, 0xc5642d3748)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87eb04ac0, 0xc5c2e3bcb0, 0xc4273bebe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87eb04ae0, 0xc4273bebe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5500, 0xc7f6e88000, 0x361, 0x6a0, 0x2fb11d9a, 0x280bcc0, 0xc7f6e88000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 515 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8282b605a4dd5ab9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc79366d220, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc79366d220, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc79366d220, 0x28006b0, 0xc889b379e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc79cf9fde0, 0xc79366d220, 0xc5dc987a90, 0xc889b37a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc79cf9fde0, 0xc5c2e3bd10, 0xc79366d220, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc79cf9fe00, 0xc79366d220, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5650, 0xc7f43a8800, 0x7d9, 0x900, 0x39f92971, 0x280bcc0, 0xc7f43a8800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 516 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x82edb2f73a8a7998)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56c992dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56c992dc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56c992dc0, 0x28006b0, 0xc8b9f127e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4de056580, 0xc56c992dc0, 0xc5e24cfa90, 0xc8b9f12898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4de056580, 0xc5c2e3bd70, 0xc56c992dc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4de0565a0, 0xc56c992dc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d57a0, 0xc6f8106000, 0xa64, 0xc00, 0x257d1583, 0x280bcc0, 0xc6f8106000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d57a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 517 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcb462b8d27e67f51)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc765b2e180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc765b2e180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc765b2e180, 0x28006b0, 0xc7528ff320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc734db8760, 0xc765b2e180, 0xc5dda01a90, 0xc7528ff3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc734db8760, 0xc5c2e3bdd0, 0xc765b2e180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc734db8780, 0xc765b2e180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d58f0, 0xc7f31b7800, 0x755, 0x900, 0x108b1e6e, 0x280bcc0, 0xc7f31b7800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d58f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 518 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4bae16497aef0365)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f0c96d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f0c96d80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f0c96d80, 0x28006b0, 0xc7c7b4d680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5d61ec5e0, 0xc7f0c96d80, 0xc5b0771a90, 0xc7c7b4d738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5d61ec5e0, 0xc5c2e3be30, 0xc7f0c96d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5d61ec640, 0xc7f0c96d80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5a40, 0xc80318a000, 0xac9, 0xc00, 0x34b9fd00, 0x280bcc0, 0xc80318a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5a40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 519 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9caca2027da04c98)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc84d3ad740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc84d3ad740, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc84d3ad740, 0x28006b0, 0xc8771d3200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc763215c20, 0xc84d3ad740, 0xc5d8489a90, 0xc8771d32b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc763215c20, 0xc5c2e3be90, 0xc84d3ad740, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc763215c40, 0xc84d3ad740, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5b90, 0xc9880f0800, 0x6e3, 0x900, 0x292cb359, 0x280bcc0, 0xc9880f0800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5b90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 520 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc60590887eae51e5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4f105da80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4f105da80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4f105da80, 0x28006b0, 0xc884322c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e5955800, 0xc4f105da80, 0xc5dbd8da90, 0xc884322d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e5955800, 0xc5c2e3bef0, 0xc4f105da80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e5955820, 0xc4f105da80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5ce0, 0xc858781800, 0x7c4, 0x900, 0x9e780d0, 0x280bcc0, 0xc858781800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5ce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 521 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5f265eec06a0f129)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b01683c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b01683c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b01683c0, 0x28006b0, 0xc6a67c10e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc565060d80, 0xc7b01683c0, 0xc5d358ba90, 0xc6a67c11b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc565060d80, 0xc5c2e3bf50, 0xc7b01683c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc565060de0, 0xc7b01683c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5e30, 0xc7b2531000, 0x803, 0x900, 0x2e22d42b, 0x280bcc0, 0xc7b2531000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5e30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 522 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3ece957337a311ea)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86f231ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86f231ae0, 0x1a753c0, 0xc5e5287a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86f231ae0, 0x28006b0, 0xc84b2defc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc88a21a980, 0xc86f231ae0, 0xc5e5287a90, 0xc84b2df078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc88a21a980, 0xc5c2e3bfb0, 0xc86f231ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc88a21a9a0, 0xc86f231ae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5b00d5f80, 0xc81a206000, 0xa3c, 0xc00, 0x1d5eb069, 0x280bcc0, 0xc81a206000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5b00d5f80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 523 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb8cccab287b20e3a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56781e120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56781e120, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56781e120, 0x28006b0, 0xc6bf600b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54fdaf640, 0xc56781e120, 0xc598677a90, 0xc6bf600bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54fdaf640, 0xc5e2454030, 0xc56781e120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54fdaf6c0, 0xc56781e120, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe40e0, 0xc8bbfb2000, 0x8c4, 0x900, 0x5c9e5c6, 0x280bcc0, 0xc8bbfb2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe40e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 524 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3c97950d8ea083cc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78e0aa140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78e0aa140, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78e0aa140, 0x28006b0, 0xc7365ecb40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc784c86ea0, 0xc78e0aa140, 0xc5c02f9a90, 0xc7365ecbf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc784c86ea0, 0xc5e2454090, 0xc78e0aa140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc784c86ec0, 0xc78e0aa140, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4230, 0xc85832b800, 0x812, 0x900, 0x6213f47, 0x280bcc0, 0xc85832b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 525 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x60a11b2abc014f0c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e6add060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e6add060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e6add060, 0x28006b0, 0xc6fea02240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6f309f640, 0xc7e6add060, 0xc5abf73a90, 0xc6fea022f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6f309f640, 0xc5e24540f0, 0xc7e6add060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6f309f660, 0xc7e6add060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4380, 0xc8c4e16000, 0xa4f, 0xc00, 0x1f254efe, 0x280bcc0, 0xc8c4e16000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 526 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfae15799b8e9134)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7ae8f2420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7ae8f2420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7ae8f2420, 0x28006b0, 0xc807fd79e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8afcfda80, 0xc7ae8f2420, 0xc5d64ffa90, 0xc807fd7aa8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8afcfda80, 0xc5e2454150, 0xc7ae8f2420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8afcfdaa0, 0xc7ae8f2420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe44d0, 0xc892b8f800, 0x6a1, 0x900, 0x23f5af27, 0x280bcc0, 0xc892b8f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe44d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 527 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x857d315f49a9d735)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc60e0c1a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc60e0c1a40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc60e0c1a40, 0x28006b0, 0xc7797cb8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7622907a0, 0xc60e0c1a40, 0xc602623a90, 0xc7797cb988)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7622907a0, 0xc5e24541b0, 0xc60e0c1a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7622907c0, 0xc60e0c1a40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4620, 0xc78ff0ac80, 0x270, 0x330, 0x2760294e, 0x280bcc0, 0xc78ff0ac80)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 528 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa06315b1ba8240f0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8131fdb80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8131fdb80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8131fdb80, 0x28006b0, 0xc72921fd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc76e77f8c0, 0xc8131fdb80, 0xc5871fda90, 0xc72921fdf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc76e77f8c0, 0xc5e2454210, 0xc8131fdb80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc76e77f8e0, 0xc8131fdb80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4770, 0xc8b8d74000, 0x932, 0xc00, 0xad729c1, 0x280bcc0, 0xc8b8d74000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 529 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xda9fe81c06a12f58)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc71faaa660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc71faaa660, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc71faaa660, 0x28006b0, 0xc4a2d967e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4acfa95c0, 0xc71faaa660, 0xc5c0291a90, 0xc4a2d96898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4acfa95c0, 0xc5e2454270, 0xc71faaa660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4acfa9640, 0xc71faaa660, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe48c0, 0xc8ac9ac000, 0x949, 0xc00, 0x15b6f8b0, 0x280bcc0, 0xc8ac9ac000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe48c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 530 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6f379cbb8846e304)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b7bb47e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b7bb47e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b7bb47e0, 0x28006b0, 0xc80679afc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81d403d40, 0xc8b7bb47e0, 0xc420f37a90, 0xc80679b078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81d403d40, 0xc5e24542d0, 0xc8b7bb47e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81d403d80, 0xc8b7bb47e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4a10, 0xc8b5447800, 0x7bf, 0x900, 0x38b3c09b, 0x280bcc0, 0xc8b5447800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4a10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 531 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd7cfd30a3e0f82c6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7996c5060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7996c5060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7996c5060, 0x28006b0, 0xc604aa8120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4dd436e60, 0xc7996c5060, 0xc5d10cda90, 0xc604aa81d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4dd436e60, 0xc5e2454330, 0xc7996c5060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4dd436ea0, 0xc7996c5060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4b60, 0xc8b4146000, 0x993, 0xc00, 0x1cff14b9, 0x280bcc0, 0xc8b4146000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4b60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 532 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x773fb2ff00756f0c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc65387da80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc65387da80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc65387da80, 0x28006b0, 0xc8b74c6a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc446693860, 0xc65387da80, 0xc598699a90, 0xc8b74c6ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc446693860, 0xc5e2454390, 0xc65387da80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4466938c0, 0xc65387da80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4cb0, 0xc8c5ff5800, 0x89d, 0x900, 0x35a4228a, 0x280bcc0, 0xc8c5ff5800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4cb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 533 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3e23a389fa0ca3f7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f4d6ac40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f4d6ac40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f4d6ac40, 0x28006b0, 0xc76b6a5560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc43b4b32c0, 0xc6f4d6ac40, 0xc605c59a90, 0xc76b6a5618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc43b4b32c0, 0xc5e24543f0, 0xc6f4d6ac40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc43b4b32e0, 0xc6f4d6ac40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4e00, 0xc87603a000, 0x9de, 0xc00, 0x274ae042, 0x280bcc0, 0xc87603a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4e00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 534 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5bd7441d0386dfbc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc829f568c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc829f568c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc829f568c0, 0x28006b0, 0xc58ccf8240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc848cd9200, 0xc829f568c0, 0xc5d605ba90, 0xc58ccf82f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc848cd9200, 0xc5e2454450, 0xc829f568c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc848cd9220, 0xc829f568c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe4f50, 0xc8829d1800, 0x75f, 0x900, 0x2cf04e96, 0x280bcc0, 0xc8829d1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe4f50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 535 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf3558101d0d189b1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8122ca980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8122ca980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8122ca980, 0x28006b0, 0xc47664d680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc50618c4e0, 0xc8122ca980, 0xc5d85dfa90, 0xc47664d738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc50618c4e0, 0xc5e24544b0, 0xc8122ca980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc50618c500, 0xc8122ca980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe50a0, 0xc8ca9e8000, 0xa21, 0xc00, 0x2d6b0da9, 0x280bcc0, 0xc8ca9e8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe50a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 536 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x362e4ed78d1f2aad)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f651e400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f651e400, 0x1a753c0, 0xc63829da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f651e400, 0x28006b0, 0xc891192480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7f299a600, 0xc7f651e400, 0xc63829da90, 0xc891192538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7f299a600, 0xc5e2454510, 0xc7f651e400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7f299a620, 0xc7f651e400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe51f0, 0xc85fa00000, 0x836, 0x900, 0x32246e5d, 0x280bcc0, 0xc85fa00000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe51f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 537 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x655067b493d01581)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e38e7a00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e38e7a00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e38e7a00, 0x28006b0, 0xc514b88360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5b3fe50c0, 0xc7e38e7a00, 0xc7da057a90, 0xc514b88418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5b3fe50c0, 0xc5e2454570, 0xc7e38e7a00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5b3fe50e0, 0xc7e38e7a00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe53b0, 0xc8cc509000, 0x900, 0x900, 0x820486a, 0x280bcc0, 0xc8cc509000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe53b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 538 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x384b7347a1f26ad4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc80b879b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc80b879b40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc80b879b40, 0x28006b0, 0xc861cd47e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ad1bbc60, 0xc80b879b40, 0xc5c4bfba90, 0xc861cd4898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ad1bbc60, 0xc5e24545d0, 0xc80b879b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7ad1bbc80, 0xc80b879b40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5500, 0xc7abd5e000, 0xb39, 0xc00, 0x82447e2, 0x280bcc0, 0xc7abd5e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 539 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xac4ae44440f2c78c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc761e1c1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc761e1c1c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc761e1c1c0, 0x28006b0, 0xc75c591b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc873eaaee0, 0xc761e1c1c0, 0xc59becfa90, 0xc75c591bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc873eaaee0, 0xc5e2454630, 0xc761e1c1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc873eaaf00, 0xc761e1c1c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5650, 0xc81d18e000, 0x956, 0xc00, 0x1bd0768f, 0x280bcc0, 0xc81d18e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 540 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7d322b5b022f16b7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc62f16b260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc62f16b260, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc62f16b260, 0x28006b0, 0xc46d4dab40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc58d27f8c0, 0xc62f16b260, 0xc5d39dba90, 0xc46d4dabf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc58d27f8c0, 0xc5e2454690, 0xc62f16b260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc58d27f8e0, 0xc62f16b260, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe57a0, 0xc7d8863800, 0x7e0, 0x900, 0x266a7e65, 0x280bcc0, 0xc7d8863800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe57a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 541 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x907b5a6569f8e464)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b2a3b540, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b2a3b540, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b2a3b540, 0x28006b0, 0xc8622f6b40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc61498d2e0, 0xc8b2a3b540, 0xc58c3e5a90, 0xc8622f6bf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc61498d2e0, 0xc5e24546f0, 0xc8b2a3b540, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc61498d300, 0xc8b2a3b540, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe58f0, 0xc7d024a000, 0x642, 0x6a0, 0x1580c0ec, 0x280bcc0, 0xc7d024a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe58f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 542 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x61814c41dec2ebe6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc84ad9e240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc84ad9e240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc84ad9e240, 0x28006b0, 0xc7e213b320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6137c35c0, 0xc84ad9e240, 0xc5e2f1ba90, 0xc7e213b3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6137c35c0, 0xc5e2454750, 0xc84ad9e240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6137c35e0, 0xc84ad9e240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5a40, 0xc7dbdac000, 0x962, 0xc00, 0x193f37f3, 0x280bcc0, 0xc7dbdac000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5a40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 543 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xaaed95ad16c29d3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5f1737d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5f1737d80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5f1737d80, 0x28006b0, 0xc6e67b4d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc65abe7c80, 0xc5f1737d80, 0xc5bfd57a90, 0xc6e67b4e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc65abe7c80, 0xc5e24547b0, 0xc5f1737d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc65abe7ca0, 0xc5f1737d80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5b90, 0xc8844e0000, 0x948, 0xc00, 0x18f89544, 0x280bcc0, 0xc8844e0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5b90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 544 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3da07e1016d0df39)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc572a15800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc572a15800, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc572a15800, 0x28006b0, 0xc980f5c480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5a59de4a0, 0xc572a15800, 0xc5d0fafa90, 0xc980f5c538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5a59de4a0, 0xc5e2454810, 0xc572a15800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5a59de4e0, 0xc572a15800, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5ce0, 0xc8b2f6f000, 0x7ec, 0x900, 0x2ff03c91, 0x280bcc0, 0xc8b2f6f000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5ce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 545 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3e839f549b76cc05)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6171a21c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6171a21c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6171a21c0, 0x28006b0, 0xc819d267e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc46f2b7600, 0xc6171a21c0, 0xc5d7daba90, 0xc819d26898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc46f2b7600, 0xc5e2454870, 0xc6171a21c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc46f2b7640, 0xc6171a21c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5e30, 0xc84588c000, 0x974, 0xc00, 0x35879227, 0x280bcc0, 0xc84588c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5e30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 546 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa9123cb0cc5e58c9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f96e0be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f96e0be0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f96e0be0, 0x28006b0, 0xc85f6f7200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc517de0320, 0xc6f96e0be0, 0xc5d9589a90, 0xc85f6f72b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc517de0320, 0xc5e24548d0, 0xc6f96e0be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc517de0380, 0xc6f96e0be0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dcfe5f80, 0xc97a308000, 0x5d6, 0x6a0, 0x20c10db4, 0x280bcc0, 0xc97a308000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dcfe5f80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 547 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfbccd5ddb57c0d27)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4aa836480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4aa836480, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4aa836480, 0x28006b0, 0xc759bc3560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc868a381e0, 0xc4aa836480, 0xc63c407a90, 0xc759bc3618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc868a381e0, 0xc5e2454930, 0xc4aa836480, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc868a38200, 0xc4aa836480, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c0e0, 0xc8aed06000, 0x980, 0xc00, 0x35231fb, 0x280bcc0, 0xc8aed06000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 548 [semacquire, 10606 minutes]:
sync.runtime_SemacquireMutex(0xc4206e9984)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206e9980)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9e4120ba58bce4e0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc53ebd2280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc53ebd2280, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc53ebd2280, 0x28006b0, 0xc476cdab40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d91e0b60, 0xc53ebd2280, 0xc5deac3a90, 0xc476cdabf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d91e0b60, 0xc5e2454990, 0xc53ebd2280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d91e0b80, 0xc53ebd2280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c230, 0xc8607ec800, 0x748, 0x900, 0x2e9663b5, 0x280bcc0, 0xc8607ec800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 549 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4e99ef79254a2356)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc75283c060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc75283c060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc75283c060, 0x28006b0, 0xc87e9898c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc82e492ac0, 0xc75283c060, 0xc5bea63a90, 0xc87e989998)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc82e492ac0, 0xc5e24549f0, 0xc75283c060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc82e492ae0, 0xc75283c060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c380, 0xc822672800, 0x76e, 0x900, 0x2d1af78d, 0x280bcc0, 0xc822672800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 550 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xcf459944d263c300)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5a6b54020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5a6b54020, 0x1a753c0, 0xc5b100da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5a6b54020, 0x28006b0, 0xc7d8593e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70acb7700, 0xc5a6b54020, 0xc5b100da90, 0xc7d8593f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70acb7700, 0xc5e2454a50, 0xc5a6b54020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70acb7740, 0xc5a6b54020, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c4d0, 0xc8ae27e000, 0xaf7, 0xc00, 0x14f98558, 0x280bcc0, 0xc8ae27e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c4d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 551 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xbb0129f9033662ae)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6658137e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6658137e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6658137e0, 0x28006b0, 0xc88ee626c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7a8922e60, 0xc6658137e0, 0xc5ab9b7a90, 0xc88ee62778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7a8922e60, 0xc5e2454ab0, 0xc6658137e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7a8922e80, 0xc6658137e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c620, 0xc98242e000, 0x9ff, 0xc00, 0x2b1834ad, 0x280bcc0, 0xc98242e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 552 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa80f1811e3a4a1c7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc62575dc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc62575dc40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc62575dc40, 0x28006b0, 0xc83081d320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc489051f80, 0xc62575dc40, 0xc5a80a9a90, 0xc83081d3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc489051f80, 0xc5e2454b10, 0xc62575dc40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc489051fe0, 0xc62575dc40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c770, 0xc845106800, 0x70a, 0x900, 0x2fb80139, 0x280bcc0, 0xc845106800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 553 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x58f123e5bf3d1b21)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7dc6ce660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7dc6ce660, 0x1a753c0, 0xc60321da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7dc6ce660, 0x28006b0, 0xc684ea4fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b87a7dc0, 0xc7dc6ce660, 0xc60321da90, 0xc684ea5078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b87a7dc0, 0xc5e2454b70, 0xc7dc6ce660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b87a7de0, 0xc7dc6ce660, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02c8c0, 0xc87cec9800, 0x7bf, 0x900, 0x45efb67, 0x280bcc0, 0xc87cec9800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02c8c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 554 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1248521c412dd2d4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7f05f0de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7f05f0de0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7f05f0de0, 0x28006b0, 0xc7c0083e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8aafbf5c0, 0xc7f05f0de0, 0xc5b692da90, 0xc7c0083f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8aafbf5c0, 0xc5e2454bd0, 0xc7f05f0de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8aafbf5e0, 0xc7f05f0de0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02ca80, 0xc8460e2000, 0x9fd, 0xc00, 0x33d28853, 0x280bcc0, 0xc8460e2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02ca80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 555 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xca2335054c379a08)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8a9944da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8a9944da0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8a9944da0, 0x28006b0, 0xc6819b65a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84a59f8c0, 0xc8a9944da0, 0xc5b011fa90, 0xc6819b6658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84a59f8c0, 0xc5e2454c30, 0xc8a9944da0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84a59f900, 0xc8a9944da0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02cbd0, 0xc7d3e62000, 0x924, 0xc00, 0x1433c29c, 0x280bcc0, 0xc7d3e62000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02cbd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 556 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf86b0f49d395531f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74d2885e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74d2885e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74d2885e0, 0x28006b0, 0xc7fb06d440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f3ce57a0, 0xc74d2885e0, 0xc5b5f49a90, 0xc7fb06d4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f3ce57a0, 0xc5e2454c90, 0xc74d2885e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f3ce57c0, 0xc74d2885e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02cd20, 0xc7d4d10a00, 0x562, 0x6a0, 0x297bd2a3, 0x280bcc0, 0xc7d4d10a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02cd20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 557 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x37b7de713592699)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5bb17fee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5bb17fee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5bb17fee0, 0x28006b0, 0xc81ade6360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80dcc7920, 0xc5bb17fee0, 0xc5e3957a90, 0xc81ade6418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80dcc7920, 0xc5e2454cf0, 0xc5bb17fee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80dcc7940, 0xc5bb17fee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02ce70, 0xc85fc27000, 0x8cf, 0x900, 0x33f5a2f8, 0x280bcc0, 0xc85fc27000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02ce70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 558 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x80b5a2dc30d3ddcc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc46d395940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc46d395940, 0x1a753c0, 0xc51c7feea8)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc46d395940, 0x28006b0, 0xc51c7feea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7577d7c20, 0xc46d395940, 0xc5abd01a90, 0xc51c7fef58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7577d7c20, 0xc5e2454d50, 0xc46d395940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7577d7c60, 0xc46d395940, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02cfc0, 0xc821f94800, 0x807, 0x900, 0xe4690e4, 0x280bcc0, 0xc821f94800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02cfc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 559 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x404f52496eb6bad0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc67eff87a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc67eff87a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc67eff87a0, 0x28006b0, 0xc70e317200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc77983cba0, 0xc67eff87a0, 0xc5979fba90, 0xc70e3172b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc77983cba0, 0xc5e2454db0, 0xc67eff87a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc77983cbe0, 0xc67eff87a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d110, 0xc6f08dc000, 0x94c, 0xc00, 0x28feacf6, 0x280bcc0, 0xc6f08dc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 560 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x25e15e4af8d3d5a3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78cebb1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78cebb1c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78cebb1c0, 0x28006b0, 0xc7a8cf26c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ba672e40, 0xc78cebb1c0, 0xc5e3af3a90, 0xc7a8cf2778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ba672e40, 0xc5e2454e10, 0xc78cebb1c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7ba672e60, 0xc78cebb1c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d260, 0xc786bc4000, 0xa54, 0xc00, 0x9c2892, 0x280bcc0, 0xc786bc4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 561 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc1b6e226f94e74ab)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7588b82e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7588b82e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7588b82e0, 0x28006b0, 0xc8af893320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80da83480, 0xc7588b82e0, 0xc605181a90, 0xc8af8933d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80da83480, 0xc5e2454e70, 0xc7588b82e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80da834a0, 0xc7588b82e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d3b0, 0xc88ec1a800, 0x7f4, 0x900, 0x26ca61a0, 0x280bcc0, 0xc88ec1a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d3b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 562 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x397e6b1f85127d37)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc434ed4940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc434ed4940, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc434ed4940, 0x28006b0, 0xc803ee2360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4b07fe420, 0xc434ed4940, 0xc5e0b99a90, 0xc803ee2428)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4b07fe420, 0xc5e2454ed0, 0xc434ed4940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4b07fe440, 0xc434ed4940, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d500, 0xc7e110a800, 0x8fc, 0x900, 0x8fb8776, 0x280bcc0, 0xc7e110a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 563 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc9350000cb9e88a4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a3467360, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a3467360, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a3467360, 0x28006b0, 0xc6d7373560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86a4c1120, 0xc7a3467360, 0xc5c4c99a90, 0xc6d7373618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86a4c1120, 0xc5e2454f30, 0xc7a3467360, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86a4c1160, 0xc7a3467360, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d650, 0xc872290000, 0x945, 0xc00, 0x216ddce3, 0x280bcc0, 0xc872290000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 564 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe7f89f0f6e09eec5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc62b84a100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc62b84a100, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc62b84a100, 0x28006b0, 0xc7fa8646c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc85f430ec0, 0xc62b84a100, 0xc601931a90, 0xc7fa864778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc85f430ec0, 0xc5e2454f90, 0xc62b84a100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc85f430ee0, 0xc62b84a100, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d7a0, 0xc7db4d4800, 0x6cb, 0x900, 0x3a560290, 0x280bcc0, 0xc7db4d4800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d7a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 565 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5ce99ff5eb293db9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc627e0a660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc627e0a660, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc627e0a660, 0x28006b0, 0xc6190ce7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc78b77bfa0, 0xc627e0a660, 0xc584987a90, 0xc6190ce898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc78b77bfa0, 0xc5e2454ff0, 0xc627e0a660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc78b77bfc0, 0xc627e0a660, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02d8f0, 0xc74c2b2000, 0x94e, 0xc00, 0x34dc2572, 0x280bcc0, 0xc74c2b2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02d8f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 566 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x43ddf22e645a3e15)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7303073c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7303073c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7303073c0, 0x28006b0, 0xc88fd86000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc71d13c9a0, 0xc7303073c0, 0xc605a71a90, 0xc88fd860b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc71d13c9a0, 0xc5e2455050, 0xc7303073c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc71d13c9e0, 0xc7303073c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02da40, 0xc838196800, 0x75a, 0x900, 0x338c0040, 0x280bcc0, 0xc838196800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02da40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 567 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf41d2eda2febab43)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc61717e860, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc61717e860, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc61717e860, 0x28006b0, 0xc563282120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc81f3ee9e0, 0xc61717e860, 0xc5b0929a90, 0xc5632821d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc81f3ee9e0, 0xc5e24550b0, 0xc61717e860, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc81f3eea00, 0xc61717e860, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02db90, 0xc857ba8000, 0x86b, 0x900, 0x215e65f3, 0x280bcc0, 0xc857ba8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02db90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 568 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe5e7b7cac5ccc3e3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6745cc760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6745cc760, 0x1a753c0, 0xc8695e7688)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6745cc760, 0x28006b0, 0xc8695e7680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc696d04ae0, 0xc6745cc760, 0xc5d9fd7a90, 0xc8695e7748)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc696d04ae0, 0xc5e2455110, 0xc6745cc760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc696d04b00, 0xc6745cc760, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02dce0, 0xc8b7568a00, 0x5a5, 0x6a0, 0x2c8c0698, 0x280bcc0, 0xc8b7568a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02dce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 569 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6b581ce3acd3cff3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc427852a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc427852a20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc427852a20, 0x28006b0, 0xc824acde60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc519f48920, 0xc427852a20, 0xc5acea1a90, 0xc824acdf28)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc519f48920, 0xc5e2455170, 0xc427852a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc519f48940, 0xc427852a20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02de30, 0xc7d2399800, 0x86e, 0x900, 0x34a5a175, 0x280bcc0, 0xc7d2399800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02de30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 570 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x97ae508d4897e9ff)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8380b2400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8380b2400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8380b2400, 0x28006b0, 0xc8b0275b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7f2c3db60, 0xc8380b2400, 0xc57e8c5a90, 0xc8b0275bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7f2c3db60, 0xc5e24551d0, 0xc8380b2400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7f2c3db80, 0xc8380b2400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dc02df80, 0xc8c6160000, 0x76a, 0x900, 0x21b50a1e, 0x280bcc0, 0xc8c6160000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dc02df80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 571 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x80725aa46b3697bd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc512eb4020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc512eb4020, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc512eb4020, 0x28006b0, 0xc6fdc9bb00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc625878320, 0xc512eb4020, 0xc5d814da90, 0xc6fdc9bbb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc625878320, 0xc5e2455230, 0xc512eb4020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc625878340, 0xc512eb4020, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc150, 0xc864886000, 0x9d4, 0xc00, 0x3699daea, 0x280bcc0, 0xc864886000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc150, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 572 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd2f1a8e9994f1981)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6b0d94680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6b0d94680, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6b0d94680, 0x28006b0, 0xc6aefd1e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e4710fa0, 0xc6b0d94680, 0xc5db7a5a90, 0xc6aefd1f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e4710fa0, 0xc5e2455290, 0xc6b0d94680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e4710fc0, 0xc6b0d94680, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc2a0, 0xc7f1f54000, 0x902, 0xc00, 0x2c61863f, 0x280bcc0, 0xc7f1f54000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc2a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 573 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1dbe9ded8f46cac)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5c8dc5c80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5c8dc5c80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5c8dc5c80, 0x28006b0, 0xc5c6dd47e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8adc8ee80, 0xc5c8dc5c80, 0xc586d71a90, 0xc5c6dd4898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8adc8ee80, 0xc5e24552f0, 0xc5c8dc5c80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8adc8eea0, 0xc5c8dc5c80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc3f0, 0xc81e083800, 0x7b3, 0x900, 0x3217e13, 0x280bcc0, 0xc81e083800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc3f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 574 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xec10cff63eb61cc6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78645a8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78645a8c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78645a8c0, 0x28006b0, 0xc5c6695680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc835a1f920, 0xc78645a8c0, 0xc6064f9a90, 0xc5c6695738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc835a1f920, 0xc5e2455350, 0xc78645a8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc835a1f940, 0xc78645a8c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc540, 0xc9871d3500, 0x62c, 0x6a0, 0x2d41314e, 0x280bcc0, 0xc9871d3500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc540, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 575 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9783dc77654e54bf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc821c73d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc821c73d00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc821c73d00, 0x28006b0, 0xc6c3206fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc448f2e900, 0xc821c73d00, 0xc5c0a93a90, 0xc6c3207078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc448f2e900, 0xc5e24553b0, 0xc821c73d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc448f2e920, 0xc821c73d00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc690, 0xc8147ec000, 0x8f1, 0x900, 0x22a8325d, 0x280bcc0, 0xc8147ec000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc690, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 576 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x954d586cccf6a7a0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc891cf3900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc891cf3900, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc891cf3900, 0x28006b0, 0xc5dbc48c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80bbdb420, 0xc891cf3900, 0xc60709ba90, 0xc5dbc48d28)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80bbdb420, 0xc5e2455410, 0xc891cf3900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80bbdb440, 0xc891cf3900, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc7e0, 0xc982d56000, 0xb2c, 0xc00, 0x27611a94, 0x280bcc0, 0xc982d56000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc7e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 577 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xbc1965a946d99295)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4b170b120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4b170b120, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4b170b120, 0x28006b0, 0xc7fabad320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80dcc8cc0, 0xc4b170b120, 0xc58414da90, 0xc7fabad3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80dcc8cc0, 0xc5e2455470, 0xc4b170b120, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80dcc8ce0, 0xc4b170b120, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fc930, 0xc7ad1d8a00, 0x5e7, 0x6a0, 0x15e7eb00, 0x280bcc0, 0xc7ad1d8a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fc930, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 578 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x246e84bb7173e84e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc64ffd3cc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc64ffd3cc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc64ffd3cc0, 0x28006b0, 0xc7b6203440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc46f729340, 0xc64ffd3cc0, 0xc5da8f7a90, 0xc7b6203508)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc46f729340, 0xc5e24554d0, 0xc64ffd3cc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc46f729380, 0xc64ffd3cc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fca80, 0xc98071e000, 0x69f, 0x6a0, 0xb088a09, 0x280bcc0, 0xc98071e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fca80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 579 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa65372dc4e687377)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6da6ea4c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6da6ea4c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6da6ea4c0, 0x28006b0, 0xc8890cca20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7ebf99060, 0xc6da6ea4c0, 0xc6032c3a90, 0xc8890ccad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7ebf99060, 0xc5e2455530, 0xc6da6ea4c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7ebf99080, 0xc6da6ea4c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fcbd0, 0xc8c92a3500, 0x4c4, 0x6a0, 0x2e07e98, 0x280bcc0, 0xc8c92a3500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fcbd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 580 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf22196601dd26a57)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6ce2428a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6ce2428a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6ce2428a0, 0x28006b0, 0xc5b65e4a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84d30cc20, 0xc6ce2428a0, 0xc59217fa90, 0xc5b65e4ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84d30cc20, 0xc5e2455590, 0xc6ce2428a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84d30cc40, 0xc6ce2428a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fcd20, 0xc8071d2000, 0xb3f, 0xc00, 0x36d9cd9c, 0x280bcc0, 0xc8071d2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fcd20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 581 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa6c1b635cbda4c0a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8622fdb40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8622fdb40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8622fdb40, 0x28006b0, 0xc5a1bb2fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc76146c560, 0xc8622fdb40, 0xc5abf71a90, 0xc5a1bb3078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc76146c560, 0xc5e24555f0, 0xc8622fdb40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc76146c580, 0xc8622fdb40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fce70, 0xc8c5fdd000, 0x7b3, 0x900, 0x1ed34dfb, 0x280bcc0, 0xc8c5fdd000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fce70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 582 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9697092865b694ff)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc54e381080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc54e381080, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc54e381080, 0x28006b0, 0xc700930900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8b7511b20, 0xc54e381080, 0xc5b9a3da90, 0xc7009309b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8b7511b20, 0xc5e2455650, 0xc54e381080, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8b7511b60, 0xc54e381080, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fcfc0, 0xc7b1d52000, 0x901, 0xc00, 0x253441de, 0x280bcc0, 0xc7b1d52000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fcfc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 583 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc801e17e1409cc12)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8afa36f00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8afa36f00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8afa36f00, 0x28006b0, 0xc804dbe900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84c166680, 0xc8afa36f00, 0xc586041a90, 0xc804dbe9b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84c166680, 0xc5e24556b0, 0xc8afa36f00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84c1666a0, 0xc8afa36f00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd110, 0xc857ebf800, 0x77d, 0x900, 0x1f80336f, 0x280bcc0, 0xc857ebf800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 584 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x46405230ab0dca07)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc427b8e4c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc427b8e4c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc427b8e4c0, 0x28006b0, 0xc6ff0b7560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc743183e60, 0xc427b8e4c0, 0xc5aad91a90, 0xc6ff0b7618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc743183e60, 0xc5e2455710, 0xc427b8e4c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc743183e80, 0xc427b8e4c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd260, 0xc7f92d4800, 0x88f, 0x900, 0x909f879, 0x280bcc0, 0xc7f92d4800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 585 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe999cdbe155f6184)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7dd6d2880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7dd6d2880, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7dd6d2880, 0x28006b0, 0xc841922c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84d3da720, 0xc7dd6d2880, 0xc5ddd47a90, 0xc841922d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84d3da720, 0xc5e2455770, 0xc7dd6d2880, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84d3da760, 0xc7dd6d2880, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd3b0, 0xc7f445d800, 0x6c0, 0x900, 0x286f68f3, 0x280bcc0, 0xc7f445d800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd3b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 586 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3ea4dc8707460a24)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc83d7381a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc83d7381a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc83d7381a0, 0x28006b0, 0xc76fa378c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc61c863280, 0xc83d7381a0, 0xc5b4687a90, 0xc76fa37978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc61c863280, 0xc5e24557d0, 0xc83d7381a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc61c8632a0, 0xc83d7381a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd500, 0xc857ab7800, 0x7fa, 0x900, 0xc75bdc9, 0x280bcc0, 0xc857ab7800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 587 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa74c68644d644d61)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc80953df20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc80953df20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc80953df20, 0x28006b0, 0xc7c421d320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc77ea7f1c0, 0xc80953df20, 0xc5e264ba90, 0xc7c421d3e8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc77ea7f1c0, 0xc5e2455830, 0xc80953df20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc77ea7f1e0, 0xc80953df20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd6c0, 0xc857eb6800, 0x7dc, 0x900, 0x13689263, 0x280bcc0, 0xc857eb6800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd6c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 588 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6b9043a0d3b69d90)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc556ed2be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc556ed2be0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc556ed2be0, 0x28006b0, 0xc7a0c8b7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc450700400, 0xc556ed2be0, 0xc5d14d5a90, 0xc7a0c8b858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc450700400, 0xc5e2455890, 0xc556ed2be0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4507004e0, 0xc556ed2be0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd810, 0xc7abac8000, 0x4d6, 0x6a0, 0x53c5de4, 0x280bcc0, 0xc7abac8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd810, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 589 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb4b5f3631d170d4e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc64330f980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc64330f980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc64330f980, 0x28006b0, 0xc83b26c000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8888be4c0, 0xc64330f980, 0xc5aee35a90, 0xc83b26c0b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8888be4c0, 0xc5e24558f0, 0xc64330f980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8888be500, 0xc64330f980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fd960, 0xc74c398000, 0xb08, 0xc00, 0xf0f27c3, 0x280bcc0, 0xc74c398000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fd960, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 590 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4ca58941a6d39ca8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc56189bf60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc56189bf60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc56189bf60, 0x28006b0, 0xc6e8a01560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83d854620, 0xc56189bf60, 0xc586a27a90, 0xc6e8a01618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83d854620, 0xc5e2455950, 0xc56189bf60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83d854640, 0xc56189bf60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fdab0, 0xc81a706000, 0x64a, 0x6a0, 0x1f3c5f5c, 0x280bcc0, 0xc81a706000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fdab0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 591 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3bde5e64e074fd2a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86e6f0440, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86e6f0440, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86e6f0440, 0x28006b0, 0xc6d5a51b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc838bd83a0, 0xc86e6f0440, 0xc5dee41a90, 0xc6d5a51bc8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc838bd83a0, 0xc5e24559b0, 0xc86e6f0440, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc838bd83c0, 0xc86e6f0440, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fdc00, 0xc7979f8000, 0x92d, 0xc00, 0x20bd091f, 0x280bcc0, 0xc7979f8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fdc00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 592 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfa72faf927fbee38)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc55957abe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc55957abe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc55957abe0, 0x28006b0, 0xc6891dea20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc55957ac00, 0xc55957abe0, 0xc6019afa90, 0xc6891deac8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc55957ac00, 0xc5e2455a10, 0xc55957abe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc55957ac60, 0xc55957abe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fdd50, 0xc8b3b72800, 0x7db, 0x900, 0x142b671a, 0x280bcc0, 0xc8b3b72800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fdd50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 593 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2a057bc223e219c5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5a44d8240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5a44d8240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5a44d8240, 0x28006b0, 0xc66358f680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54d934b60, 0xc5a44d8240, 0xc5db22fa90, 0xc66358f738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54d934b60, 0xc5e2455a70, 0xc5a44d8240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54d934b80, 0xc5a44d8240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5d92fdea0, 0xc97fa06000, 0x96f, 0xc00, 0x288f388d, 0x280bcc0, 0xc97fa06000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5d92fdea0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 594 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8acc28ddf187b8bf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7b71910a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7b71910a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7b71910a0, 0x28006b0, 0xc5e31ccb40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc72a3a3f40, 0xc7b71910a0, 0xc81fa1ba90, 0xc5e31ccbf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc72a3a3f40, 0xc5e2455ad0, 0xc7b71910a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc72a3a3f60, 0xc7b71910a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392000, 0xc8cc101000, 0x72e, 0x900, 0x1176077c, 0x280bcc0, 0xc8cc101000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392000, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 595 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x280fb401bb8c47ec)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e1fd1280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e1fd1280, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e1fd1280, 0x28006b0, 0xc70159ed80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5181d6500, 0xc4e1fd1280, 0xc591cf3a90, 0xc70159ee38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5181d6500, 0xc5e2455b30, 0xc4e1fd1280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5181d6520, 0xc4e1fd1280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392150, 0xc7e88d4000, 0x39c, 0x6a0, 0x288fc9c1, 0x280bcc0, 0xc7e88d4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392150, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 596 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x58c2cfd19dc2bdd0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5eaaf8420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5eaaf8420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5eaaf8420, 0x28006b0, 0xc81ce93200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc58db52ee0, 0xc5eaaf8420, 0xc5ac31ba90, 0xc81ce932b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc58db52ee0, 0xc5e2455b90, 0xc5eaaf8420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc58db52f20, 0xc5eaaf8420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df3922a0, 0xc7d77d2000, 0x63a, 0x6a0, 0x2c6e7110, 0x280bcc0, 0xc7d77d2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df3922a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 597 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5815ecc028b05e02)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc54caf8e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc54caf8e40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc54caf8e40, 0x28006b0, 0xc7a4350a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc570e619c0, 0xc54caf8e40, 0xc5e3657a90, 0xc7a4350ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc570e619c0, 0xc5e2455bf0, 0xc54caf8e40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc570e619e0, 0xc54caf8e40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df3923f0, 0xc8b4baf000, 0x8b3, 0x900, 0x13fda3bd, 0x280bcc0, 0xc8b4baf000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df3923f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 598 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x920b2b1065ea1f1f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5e87b0680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5e87b0680, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5e87b0680, 0x28006b0, 0xc75890dd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc79a0cb3e0, 0xc5e87b0680, 0xc5ac9b5a90, 0xc75890ddf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc79a0cb3e0, 0xc5e2455c50, 0xc5e87b0680, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc79a0cb400, 0xc5e87b0680, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392540, 0xc790c67500, 0x691, 0x6a0, 0x528309f, 0x280bcc0, 0xc790c67500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392540, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 599 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8bb6eac644ca562b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7d39fe1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7d39fe1e0, 0x1a753c0, 0xc8726ad0f8)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7d39fe1e0, 0x28006b0, 0xc8726ad0e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc849725ac0, 0xc7d39fe1e0, 0xc601c7ba90, 0xc8726ad1a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc849725ac0, 0xc5e2455cb0, 0xc7d39fe1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc849725ae0, 0xc7d39fe1e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392690, 0xc7d527d500, 0x4ba, 0x6a0, 0x17fdc506, 0x280bcc0, 0xc7d527d500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392690, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 600 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe6001fba1472745)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6be482200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6be482200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6be482200, 0x28006b0, 0xc811468480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e872c0a0, 0xc6be482200, 0xc5a4a61a90, 0xc811468538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e872c0a0, 0xc5e2455d10, 0xc6be482200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e872c0c0, 0xc6be482200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df3927e0, 0xc7c3f44000, 0x94b, 0xc00, 0xd57f0d7, 0x280bcc0, 0xc7c3f44000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df3927e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 601 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdc4107862d054540)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc662466200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc662466200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc662466200, 0x28006b0, 0xc60f12f320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc68bcb8f40, 0xc662466200, 0xc601efda90, 0xc60f12f3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc68bcb8f40, 0xc5e2455d70, 0xc662466200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc68bcb8f60, 0xc662466200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392930, 0xc97ff9f000, 0x7de, 0x900, 0x1b668c8a, 0x280bcc0, 0xc97ff9f000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392930, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 602 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6a844cf1c306bccd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc875be3ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc875be3ce0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc875be3ce0, 0x28006b0, 0xc599e49200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc648c519c0, 0xc875be3ce0, 0xc5c0a83a90, 0xc599e492b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc648c519c0, 0xc5e2455dd0, 0xc875be3ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc648c51a20, 0xc875be3ce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392a80, 0xc87d6d2000, 0x967, 0xc00, 0x1b4ebe71, 0x280bcc0, 0xc87d6d2000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392a80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 603 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x39849dbfc2c49c41)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4b4a2a580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4b4a2a580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4b4a2a580, 0x28006b0, 0xc5bb7d8480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4a7b5e260, 0xc4b4a2a580, 0xc596597a90, 0xc5bb7d8538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4a7b5e260, 0xc5e2455e30, 0xc4b4a2a580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4a7b5e280, 0xc4b4a2a580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392bd0, 0xc8cc124800, 0x801, 0x900, 0x38c6293e, 0x280bcc0, 0xc8cc124800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392bd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 604 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x876d72d3d1f2f4d8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6ad213560, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6ad213560, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6ad213560, 0x28006b0, 0xc626126a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc554d3cce0, 0xc6ad213560, 0xc6034efa90, 0xc626126ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc554d3cce0, 0xc5e2455e90, 0xc6ad213560, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc554d3cd00, 0xc6ad213560, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392d20, 0xc7f50ba000, 0x96b, 0xc00, 0x387fea62, 0x280bcc0, 0xc7f50ba000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392d20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 605 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x81ba78ee181eb8d7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc526386640, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc526386640, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc526386640, 0x28006b0, 0xc982d15c20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc73e1245a0, 0xc526386640, 0xc5cb31da90, 0xc982d15cd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc73e1245a0, 0xc5e2455ef0, 0xc526386640, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc73e1245c0, 0xc526386640, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df392ee0, 0xc7d8570000, 0x876, 0x900, 0x7ff51ae, 0x280bcc0, 0xc7d8570000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df392ee0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 606 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa82bd586f8d5f290)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc69d6b68a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc69d6b68a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc69d6b68a0, 0x28006b0, 0xc7e577d560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6b2fe22e0, 0xc69d6b68a0, 0xc5ace9da90, 0xc7e577d618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6b2fe22e0, 0xc5e2455f50, 0xc69d6b68a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6b2fe2300, 0xc69d6b68a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393030, 0xc8bb6c8000, 0x7e9, 0x900, 0x1d231fb0, 0x280bcc0, 0xc8bb6c8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393030, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 607 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x49baffde7854a66f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b5107ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b5107ec0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b5107ec0, 0x28006b0, 0xc7044fa240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8aa08a0c0, 0xc8b5107ec0, 0xc597909a90, 0xc7044fa308)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8aa08a0c0, 0xc5e2455fb0, 0xc8b5107ec0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8aa08a0e0, 0xc8b5107ec0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393180, 0xc97fc79000, 0x8d2, 0x900, 0x24366829, 0x280bcc0, 0xc97fc79000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393180, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 608 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x969162d7a07efbcc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc532418ac0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc532418ac0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc532418ac0, 0x28006b0, 0xc86170c5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b93200a0, 0xc532418ac0, 0xc58eef9a90, 0xc86170c658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b93200a0, 0xc5ce9e2030, 0xc532418ac0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b93200c0, 0xc532418ac0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df3932d0, 0xc984f0e000, 0x6d0, 0x900, 0x73d42fd, 0x280bcc0, 0xc984f0e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df3932d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 609 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8706a180f211daf4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8054747e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8054747e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8054747e0, 0x28006b0, 0xc6ce54dd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc77be93400, 0xc8054747e0, 0xc5ac79ba90, 0xc6ce54ddf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc77be93400, 0xc5ce9e2090, 0xc8054747e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc77be93420, 0xc8054747e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393420, 0xc8b30f5500, 0x644, 0x6a0, 0x56866f8, 0x280bcc0, 0xc8b30f5500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393420, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 610 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9cf5679baa80bcfe)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc866a58d40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc866a58d40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc866a58d40, 0x28006b0, 0xc6fc20dd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc74606c2c0, 0xc866a58d40, 0xc587f5ba90, 0xc6fc20de08)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc74606c2c0, 0xc5ce9e20f0, 0xc866a58d40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc74606c320, 0xc866a58d40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393570, 0xc881eb6000, 0xade, 0xc00, 0xb605475, 0x280bcc0, 0xc881eb6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393570, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 611 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6075a837b14976bd)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8af880760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8af880760, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8af880760, 0x28006b0, 0xc8aad0cd80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6240dd080, 0xc8af880760, 0xc5b5a63a90, 0xc8aad0ce38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6240dd080, 0xc5ce9e2150, 0xc8af880760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6240dd120, 0xc8af880760, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df3936c0, 0xc7d70c4800, 0x6e7, 0x900, 0x356d54fa, 0x280bcc0, 0xc7d70c4800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df3936c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 612 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x22b2c68e107f3c83)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74c87ae60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74c87ae60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74c87ae60, 0x28006b0, 0xc789e078c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80159a6c0, 0xc74c87ae60, 0xc5b4223a90, 0xc789e07978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80159a6c0, 0xc5ce9e21b0, 0xc74c87ae60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80159a6e0, 0xc74c87ae60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393810, 0xc986a2e800, 0x7fc, 0x900, 0xb459eb2, 0x280bcc0, 0xc986a2e800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393810, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 613 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xfaade4e6a88f561b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88a06fa00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88a06fa00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88a06fa00, 0x28006b0, 0xc6cf505440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d6e10920, 0xc88a06fa00, 0xc5de48ba90, 0xc6cf5054f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d6e10920, 0xc5ce9e2210, 0xc88a06fa00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d6e10940, 0xc88a06fa00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393960, 0xc8b179a800, 0x80b, 0x900, 0x3569f0d, 0x280bcc0, 0xc8b179a800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393960, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 614 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7eebd9d41252c600)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6be589ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6be589ae0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6be589ae0, 0x28006b0, 0xc6a9fbac60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7943d0fe0, 0xc6be589ae0, 0xc5989f3a90, 0xc6a9fbad18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7943d0fe0, 0xc5ce9e2270, 0xc6be589ae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7943d1000, 0xc6be589ae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393ab0, 0xc81f016000, 0x939, 0xc00, 0xb7c2f77, 0x280bcc0, 0xc81f016000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393ab0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 615 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5734aa0a9d93eefc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6b8e1a0c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6b8e1a0c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6b8e1a0c0, 0x28006b0, 0xc6a2c29e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc62040bb80, 0xc6b8e1a0c0, 0xc5af307a90, 0xc6a2c29f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc62040bb80, 0xc5ce9e22d0, 0xc6b8e1a0c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc62040bbc0, 0xc6b8e1a0c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393c00, 0xc7f8060a00, 0x560, 0x6a0, 0x2a7bb53d, 0x280bcc0, 0xc7f8060a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393c00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 616 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x29284584ddc11786)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc87dc89ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc87dc89ce0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc87dc89ce0, 0x28006b0, 0xc7c53c1e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e80b6e80, 0xc87dc89ce0, 0xc7df921a90, 0xc7c53c1f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e80b6e80, 0xc5ce9e2330, 0xc87dc89ce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e80b6ec0, 0xc87dc89ce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393d50, 0xc8b51dea00, 0x622, 0x6a0, 0x23002f15, 0x280bcc0, 0xc8b51dea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393d50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 617 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x535ee732ab881d5f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc806186d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc806186d20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc806186d20, 0x28006b0, 0xc7150259e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc591bc26a0, 0xc806186d20, 0xc5b84dfa90, 0xc715025a98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc591bc26a0, 0xc5ce9e2390, 0xc806186d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc591bc26c0, 0xc806186d20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5df393f10, 0xc7d4eb0000, 0x8a6, 0x900, 0x2276802, 0x280bcc0, 0xc7d4eb0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5df393f10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 618 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x553310fee102a9db)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc63f8df520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc63f8df520, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc63f8df520, 0x28006b0, 0xc864372000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4e00cb800, 0xc63f8df520, 0xc5d10c7a90, 0xc8643720b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4e00cb800, 0xc5ce9e23f0, 0xc63f8df520, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4e00cb820, 0xc63f8df520, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac070, 0xc580ff6000, 0xab9, 0xc00, 0x3ab01d2, 0x280bcc0, 0xc580ff6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac070, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 619 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8182d232eea3b6e3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc862ac9400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc862ac9400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc862ac9400, 0x28006b0, 0xc7dd0d0d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7d56d9800, 0xc862ac9400, 0xc601c79a90, 0xc7dd0d0e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7d56d9800, 0xc5ce9e2450, 0xc862ac9400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7d56d9860, 0xc862ac9400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac1c0, 0xc822716000, 0x842, 0x900, 0x181f6632, 0x280bcc0, 0xc822716000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac1c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 620 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4b9ed4c74dba0f2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e5686260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e5686260, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e5686260, 0x28006b0, 0xc548de6120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7e3cb5460, 0xc7e5686260, 0xc5bf309a90, 0xc548de61e8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7e3cb5460, 0xc5ce9e24b0, 0xc7e5686260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7e3cb5480, 0xc7e5686260, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac310, 0xc846248a00, 0x565, 0x6a0, 0x1815d7ee, 0x280bcc0, 0xc846248a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac310, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 621 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdf862a1bc00b95d3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a5482e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a5482e00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a5482e00, 0x28006b0, 0xc6acf24d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc64bf7a900, 0xc7a5482e00, 0xc5b7e53a90, 0xc6acf24e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc64bf7a900, 0xc5ce9e2510, 0xc7a5482e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc64bf7a940, 0xc7a5482e00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac460, 0xc981575000, 0x6e4, 0x900, 0x260d602f, 0x280bcc0, 0xc981575000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac460, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 622 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x756d86ec41e206eb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4d4cc5d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4d4cc5d80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4d4cc5d80, 0x28006b0, 0xc5db00d8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70a38e3a0, 0xc4d4cc5d80, 0xc604873a90, 0xc5db00d978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70a38e3a0, 0xc5ce9e2570, 0xc4d4cc5d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70a38e3e0, 0xc4d4cc5d80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac5b0, 0xc87e55a000, 0x899, 0x900, 0x650fc5e, 0x280bcc0, 0xc87e55a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac5b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 623 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x187f81dd2a2093da)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81048a700, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81048a700, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81048a700, 0x28006b0, 0xc7bb4db200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70ac7cd60, 0xc81048a700, 0xc58603fa90, 0xc7bb4db2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70ac7cd60, 0xc5ce9e25d0, 0xc81048a700, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70ac7cd80, 0xc81048a700, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac700, 0xc857ebb000, 0x778, 0x900, 0x27190e52, 0x280bcc0, 0xc857ebb000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac700, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 624 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8fae44b538056373)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc839ccfb00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc839ccfb00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc839ccfb00, 0x28006b0, 0xc83e031b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6db6bfb60, 0xc839ccfb00, 0xc5fba0fa90, 0xc83e031bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6db6bfb60, 0xc5ce9e2630, 0xc839ccfb00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6db6bfbc0, 0xc839ccfb00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac850, 0xc7db4d0000, 0x740, 0x900, 0x289bf5bb, 0x280bcc0, 0xc7db4d0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac850, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 625 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x62171a34d6812d50)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5539b8960, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5539b8960, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5539b8960, 0x28006b0, 0xc8b8cbb440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5525d1bc0, 0xc5539b8960, 0xc5de193a90, 0xc8b8cbb4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5525d1bc0, 0xc5ce9e2690, 0xc5539b8960, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5525d1be0, 0xc5539b8960, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debac9a0, 0xc8b543a000, 0x7ef, 0x900, 0x338bfb4b, 0x280bcc0, 0xc8b543a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debac9a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 626 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6fd2f2e445a73ac5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc747820020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc747820020, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc747820020, 0x28006b0, 0xc823828240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8638d1680, 0xc747820020, 0xc5de195a90, 0xc8238282f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8638d1680, 0xc5ce9e26f0, 0xc747820020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8638d16c0, 0xc747820020, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debacaf0, 0xc7a6f35500, 0x616, 0x6a0, 0x23f0242d, 0x280bcc0, 0xc7a6f35500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debacaf0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 627 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd941b12737c94e8f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6aa455140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6aa455140, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6aa455140, 0x28006b0, 0xc5f0850240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83de07380, 0xc6aa455140, 0xc5ac799a90, 0xc5f08502f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83de07380, 0xc5ce9e2750, 0xc6aa455140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83de073a0, 0xc6aa455140, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debacc40, 0xc7b25ce000, 0x9db, 0xc00, 0x273414ab, 0x280bcc0, 0xc7b25ce000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debacc40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 628 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5b2da1491c7f9e0c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc568abe840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc568abe840, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc568abe840, 0x28006b0, 0xc789d8aea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d99c70c0, 0xc568abe840, 0xc582267a90, 0xc789d8af58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d99c70c0, 0xc5ce9e27b0, 0xc568abe840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d99c70e0, 0xc568abe840, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debacd90, 0xc8b1aa0000, 0xabb, 0xc00, 0xbfafe55, 0x280bcc0, 0xc8b1aa0000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debacd90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 629 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2bfec8ff6343947f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6b34f2840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6b34f2840, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6b34f2840, 0x28006b0, 0xc6cb7eea20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc49d13ab40, 0xc6b34f2840, 0xc5b1409a90, 0xc6cb7eead8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc49d13ab40, 0xc5ce9e2810, 0xc6b34f2840, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc49d13ab60, 0xc6b34f2840, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debacf50, 0xc85f9e2a00, 0x671, 0x6a0, 0x113c627b, 0x280bcc0, 0xc85f9e2a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debacf50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 630 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x90d39100d91959fc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81d6aa8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81d6aa8c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81d6aa8c0, 0x28006b0, 0xc80361ec60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83473cf40, 0xc81d6aa8c0, 0xc5bb94fa90, 0xc80361ed18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83473cf40, 0xc5ce9e2870, 0xc81d6aa8c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83473cf80, 0xc81d6aa8c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad0a0, 0xc7b391c000, 0x825, 0x900, 0x228e2dde, 0x280bcc0, 0xc7b391c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad0a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 631 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe9fd61dc340fc436)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6b4a43ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6b4a43ea0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6b4a43ea0, 0x28006b0, 0xc70b624480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc58e303080, 0xc6b4a43ea0, 0xc594621a90, 0xc70b624538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc58e303080, 0xc5ce9e28d0, 0xc6b4a43ea0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc58e3030a0, 0xc6b4a43ea0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad1f0, 0xc7f4450000, 0x754, 0x900, 0x34c54ec2, 0x280bcc0, 0xc7f4450000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad1f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 632 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x55c958bae88e6ff7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7a339df40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7a339df40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7a339df40, 0x28006b0, 0xc8077a1680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc70506f3e0, 0xc7a339df40, 0xc5d78b7a90, 0xc8077a1738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc70506f3e0, 0xc5ce9e2930, 0xc7a339df40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc70506f420, 0xc7a339df40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad340, 0xc7e97c1000, 0x73c, 0x900, 0x3d10390, 0x280bcc0, 0xc7e97c1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 633 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x12ac30df72aeb14b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc79553a920, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc79553a920, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc79553a920, 0x28006b0, 0xc7dc16bc20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc72032d6e0, 0xc79553a920, 0xc5e41afa90, 0xc7dc16bcd8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc72032d6e0, 0xc5ce9e2990, 0xc79553a920, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc72032d700, 0xc79553a920, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad490, 0xc848093800, 0x82a, 0x900, 0x28d2596a, 0x280bcc0, 0xc848093800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 634 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x93bd2d178ca6ebc3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc827f87b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc827f87b40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc827f87b40, 0x28006b0, 0xc76bc12d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc46038d2c0, 0xc827f87b40, 0xc5dad67a90, 0xc76bc12e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc46038d2c0, 0xc5ce9e29f0, 0xc827f87b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc46038d300, 0xc827f87b40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad650, 0xc8c9460000, 0x9fb, 0xc00, 0xfdec824, 0x280bcc0, 0xc8c9460000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 635 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x885a8a5efde5b878)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc46031e3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc46031e3e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc46031e3e0, 0x28006b0, 0xc7f2b92480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc51df92680, 0xc46031e3e0, 0xc586045a90, 0xc7f2b92548)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc51df92680, 0xc5ce9e2a50, 0xc46031e3e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc51df926e0, 0xc46031e3e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad7a0, 0xc8906bd800, 0x827, 0x900, 0x360f36e8, 0x280bcc0, 0xc8906bd800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad7a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 636 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xba2eef62e028feaf)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74d4657a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74d4657a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74d4657a0, 0x28006b0, 0xc834ce3e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5cbb5f1a0, 0xc74d4657a0, 0xc5c2495a90, 0xc834ce3f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5cbb5f1a0, 0xc5ce9e2ab0, 0xc74d4657a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5cbb5f1c0, 0xc74d4657a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debad8f0, 0xc8b7590000, 0x997, 0xc00, 0x9e7900c, 0x280bcc0, 0xc8b7590000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debad8f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 637 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe9cb83e4b9ae332f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc61c613780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc61c613780, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc61c613780, 0x28006b0, 0xc797edf680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5754614c0, 0xc61c613780, 0xc59c2ada90, 0xc797edf738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5754614c0, 0xc5ce9e2b10, 0xc61c613780, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5754614e0, 0xc61c613780, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debada40, 0xc7d0772000, 0x691, 0x6a0, 0x3e1fa0f, 0x280bcc0, 0xc7d0772000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debada40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 638 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2be5508f5416d7a3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5c00d5720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5c00d5720, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5c00d5720, 0x28006b0, 0xc7f7128c60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc78a8acee0, 0xc5c00d5720, 0xc59fc49a90, 0xc7f7128d18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc78a8acee0, 0xc5ce9e2b70, 0xc5c00d5720, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc78a8acf00, 0xc5c00d5720, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debadb90, 0xc8b5416000, 0xa54, 0xc00, 0x2cfb934d, 0x280bcc0, 0xc8b5416000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debadb90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 639 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debadce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:440 +0x34b
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 640 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x306dd5c43f12f2b7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc517f92fe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc517f92fe0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc517f92fe0, 0x28006b0, 0xc7e681bb00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5e81a5540, 0xc517f92fe0, 0xc6032ffa90, 0xc7e681bbc8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5e81a5540, 0xc5ce9e2c30, 0xc517f92fe0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5e81a5560, 0xc517f92fe0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debade30, 0xc7f8030800, 0x778, 0x900, 0x38cb94c, 0x280bcc0, 0xc7f8030800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debade30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 641 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa7a2a6a2dfa628e1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc870fa1d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc870fa1d00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc870fa1d00, 0x28006b0, 0xc6d5602ea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc555a566c0, 0xc870fa1d00, 0xc5a3153a90, 0xc6d5602f58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc555a566c0, 0xc5ce9e2c90, 0xc870fa1d00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc555a566e0, 0xc870fa1d00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5debadf80, 0xc821b0b800, 0x8af, 0x900, 0x65d843b, 0x280bcc0, 0xc821b0b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5debadf80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 642 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x71331187714808ae)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc788d25b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc788d25b00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc788d25b00, 0x28006b0, 0xc7e8d3a360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc56f258a60, 0xc788d25b00, 0xc5d2e39a90, 0xc7e8d3a418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc56f258a60, 0xc5ce9e2cf0, 0xc788d25b00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc56f258a80, 0xc788d25b00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c0e0, 0xc8c5d2c000, 0x425, 0x6a0, 0x1fbf073b, 0x280bcc0, 0xc8c5d2c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 643 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3b0dda9564db718d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7df899300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7df899300, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7df899300, 0x28006b0, 0xc8af412360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87a090b60, 0xc7df899300, 0xc603f77a90, 0xc8af412418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87a090b60, 0xc5ce9e2d50, 0xc7df899300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87a090b80, 0xc7df899300, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c2a0, 0xc8b3e1f800, 0x73b, 0x900, 0x1a14d9ba, 0x280bcc0, 0xc8b3e1f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c2a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 644 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc06ff1b0eeb65663)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82c737c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82c737c60, 0x1a753c0, 0xc602439a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82c737c60, 0x28006b0, 0xc807a43b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc767762cc0, 0xc82c737c60, 0xc602439a90, 0xc807a43bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc767762cc0, 0xc5ce9e2db0, 0xc82c737c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc767762ce0, 0xc82c737c60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c3f0, 0xc830a40c80, 0x224, 0x330, 0x281ec66a, 0x280bcc0, 0xc830a40c80)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c3f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 645 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x80bc8b9db5590f21)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc629115b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc629115b80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc629115b80, 0x28006b0, 0xc783f8aea0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc72cecdc20, 0xc629115b80, 0xc5d4d05a90, 0xc783f8af58)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc72cecdc20, 0xc5ce9e2e10, 0xc629115b80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc72cecdc40, 0xc629115b80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c540, 0xc7d7a28000, 0x531, 0x6a0, 0x1433af28, 0x280bcc0, 0xc7d7a28000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c540, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 646 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe2988d6222703773)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5677eff20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5677eff20, 0x1a753c0, 0xc592397a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5677eff20, 0x28006b0, 0xc863421e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc792c4f960, 0xc5677eff20, 0xc592397a90, 0xc863421f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc792c4f960, 0xc5ce9e2e70, 0xc5677eff20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc792c4f9a0, 0xc5677eff20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c690, 0xc8607e8000, 0x89d, 0x900, 0x10e563ca, 0x280bcc0, 0xc8607e8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c690, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 647 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd4928083948c6c28)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8400c7d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8400c7d20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8400c7d20, 0x28006b0, 0xc8b75d98c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6ed435280, 0xc8400c7d20, 0xc5adc7ba90, 0xc8b75d9978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6ed435280, 0xc5ce9e2ed0, 0xc8400c7d20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6ed4352a0, 0xc8400c7d20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c7e0, 0xc8aea16800, 0x8ad, 0x900, 0x3b028e6, 0x280bcc0, 0xc8aea16800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c7e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 648 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x21643ec79f9dae99)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4443dbde0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4443dbde0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4443dbde0, 0x28006b0, 0xc7ac0bfd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc451fe6180, 0xc4443dbde0, 0xc5c0a95a90, 0xc7ac0bfdf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc451fe6180, 0xc5ce9e2f30, 0xc4443dbde0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc451fe61c0, 0xc4443dbde0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0c930, 0xc7f39b3800, 0x74b, 0x900, 0x23259edf, 0x280bcc0, 0xc7f39b3800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0c930, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 649 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5ab023341baab0ff)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc87589ed00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc87589ed00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc87589ed00, 0x28006b0, 0xc727fc77a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc87ba95a60, 0xc87589ed00, 0xc5b468ba90, 0xc727fc7858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc87ba95a60, 0xc5ce9e2f90, 0xc87589ed00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc87ba95a80, 0xc87589ed00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0ca80, 0xc9878eb000, 0x848, 0x900, 0xb398d62, 0x280bcc0, 0xc9878eb000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0ca80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 650 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xaf5e3166c66c6e6f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81d142060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81d142060, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81d142060, 0x28006b0, 0xc8495fb9e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f8515980, 0xc81d142060, 0xc5a2db3a90, 0xc8495fba98)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f8515980, 0xc5ce9e2ff0, 0xc81d142060, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f85159a0, 0xc81d142060, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0cbd0, 0xc7d6c0f800, 0x80f, 0x900, 0x28a9600d, 0x280bcc0, 0xc7d6c0f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0cbd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 651 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x27f8e080a766d825)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5ee40c1a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5ee40c1a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5ee40c1a0, 0x28006b0, 0xc87692a360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6e3a278c0, 0xc5ee40c1a0, 0xc7e1eefa90, 0xc87692a418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6e3a278c0, 0xc5ce9e3050, 0xc5ee40c1a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6e3a278e0, 0xc5ee40c1a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0cd20, 0xc83fd3e000, 0x96e, 0xc00, 0x129d51a9, 0x280bcc0, 0xc83fd3e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0cd20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 652 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc541b49dc86aead8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4893f8300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4893f8300, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4893f8300, 0x28006b0, 0xc86d7f9d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc43582c100, 0xc4893f8300, 0xc5e33d5a90, 0xc86d7f9df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc43582c100, 0xc5ce9e30b0, 0xc4893f8300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc43582c120, 0xc4893f8300, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0ce70, 0xc7ad2eaa00, 0x546, 0x6a0, 0x2c804f62, 0x280bcc0, 0xc7ad2eaa00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0ce70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 653 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8b7255308578086f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4f3b29180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4f3b29180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4f3b29180, 0x28006b0, 0xc627ed3680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc834d6ba20, 0xc4f3b29180, 0xc5d99d3a90, 0xc627ed3738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc834d6ba20, 0xc5ce9e3110, 0xc4f3b29180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc834d6ba40, 0xc4f3b29180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0cfc0, 0xc8b5b00a00, 0x66f, 0x6a0, 0x1722cb40, 0x280bcc0, 0xc8b5b00a00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0cfc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 654 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x650d82179738bc4a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5f7bc07e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5f7bc07e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5f7bc07e0, 0x28006b0, 0xc6b9735d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6871398c0, 0xc5f7bc07e0, 0xc5b0781a90, 0xc6b9735df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6871398c0, 0xc5ce9e3170, 0xc5f7bc07e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6871398e0, 0xc5f7bc07e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d110, 0xc82280d800, 0x6f5, 0x900, 0x4c3bfe0, 0x280bcc0, 0xc82280d800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 655 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4607b261d7387b0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4a29e28c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4a29e28c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4a29e28c0, 0x28006b0, 0xc8643e86c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc86b4c47c0, 0xc4a29e28c0, 0xc5c4c9fa90, 0xc8643e8778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc86b4c47c0, 0xc5ce9e31d0, 0xc4a29e28c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc86b4c47e0, 0xc4a29e28c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d260, 0xc987731500, 0x5d8, 0x6a0, 0x2221798e, 0x280bcc0, 0xc987731500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 656 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa1d473e813609cb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc54f797320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc54f797320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc54f797320, 0x28006b0, 0xc7b071a480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc864d7c480, 0xc54f797320, 0xc5cd873a90, 0xc7b071a538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc864d7c480, 0xc5ce9e3230, 0xc54f797320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc864d7c4a0, 0xc54f797320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d3b0, 0xc7f7503800, 0x8a1, 0x900, 0x28ddbacd, 0x280bcc0, 0xc7f7503800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d3b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 657 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206cebc4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206cebc0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb5d10606e0e6841)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc880552420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc880552420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc880552420, 0x28006b0, 0xc6d600f0e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc69cf63280, 0xc880552420, 0xc57e8c7a90, 0xc6d600f1a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc69cf63280, 0xc5ce9e3290, 0xc880552420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc69cf632a0, 0xc880552420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d500, 0xc8b782ea00, 0x598, 0x6a0, 0x1e7c0268, 0x280bcc0, 0xc8b782ea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 658 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb00d08cc72cd6be4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc745ad8a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc745ad8a40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc745ad8a40, 0x28006b0, 0xc861b49b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc45e3f2b80, 0xc745ad8a40, 0xc5b5a5fa90, 0xc861b49bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc45e3f2b80, 0xc5ce9e32f0, 0xc745ad8a40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc45e3f2c40, 0xc745ad8a40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d650, 0xc980db6000, 0x70f, 0x900, 0x33f37d7e, 0x280bcc0, 0xc980db6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 659 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xee5faffc8a410ddc)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc60f306fa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc60f306fa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc60f306fa0, 0x28006b0, 0xc981d62120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7d1da80c0, 0xc60f306fa0, 0xc5b5a61a90, 0xc981d621d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7d1da80c0, 0xc5ce9e3350, 0xc60f306fa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7d1da80e0, 0xc60f306fa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d7a0, 0xc8a861f000, 0x7f4, 0x900, 0x2d77639c, 0x280bcc0, 0xc8a861f000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d7a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 660 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x993c40475bd08a72)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6c6e9bbc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6c6e9bbc0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6c6e9bbc0, 0x28006b0, 0xc47bdc1200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc78ecdb940, 0xc6c6e9bbc0, 0xc5e05c9a90, 0xc47bdc12b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc78ecdb940, 0xc5ce9e33b0, 0xc6c6e9bbc0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc78ecdb960, 0xc6c6e9bbc0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0d8f0, 0xc7d4d0a000, 0x419, 0x6a0, 0xaeb1200, 0x280bcc0, 0xc7d4d0a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0d8f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 661 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x95f67fbf7514fb4e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5a3994020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5a3994020, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5a3994020, 0x28006b0, 0xc65f5b3200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc50e674420, 0xc5a3994020, 0xc5a314fa90, 0xc65f5b32c8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc50e674420, 0xc5ce9e3410, 0xc5a3994020, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc50e674460, 0xc5a3994020, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0da40, 0xc7f6e8ea00, 0x61d, 0x6a0, 0x1e1c3c7, 0x280bcc0, 0xc7f6e8ea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0da40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 662 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xdccf3897922812de)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc811cee000, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc811cee000, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc811cee000, 0x28006b0, 0xc636eea480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc74e710840, 0xc811cee000, 0xc5b86cba90, 0xc636eea538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc74e710840, 0xc5ce9e3470, 0xc811cee000, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc74e710860, 0xc811cee000, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0db90, 0xc8ad82ca00, 0x5f0, 0x6a0, 0x25f01e8, 0x280bcc0, 0xc8ad82ca00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0db90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 663 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x23849ecbea45ace)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8378e6760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8378e6760, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8378e6760, 0x28006b0, 0xc5c07fd680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6f21f4d80, 0xc8378e6760, 0xc57ec0ba90, 0xc5c07fd738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6f21f4d80, 0xc5ce9e34d0, 0xc8378e6760, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6f21f4da0, 0xc8378e6760, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0dce0, 0xc980934800, 0x7b7, 0x900, 0x35dbabc, 0x280bcc0, 0xc980934800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0dce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 664 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5110ce5bb52e34b6)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc77f0dc900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc77f0dc900, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc77f0dc900, 0x28006b0, 0xc7e4404120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7bb4ddf60, 0xc77f0dc900, 0xc603f79a90, 0xc7e44041d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7bb4ddf60, 0xc5ce9e3530, 0xc77f0dc900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7bb4ddfa0, 0xc77f0dc900, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0de30, 0xc9806f9000, 0x813, 0x900, 0x1db968d, 0x280bcc0, 0xc9806f9000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0de30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 665 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3eeefcf21a643773)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78c466580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78c466580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78c466580, 0x28006b0, 0xc649500a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6e4649a60, 0xc78c466580, 0xc57e7bda90, 0xc649500ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6e4649a60, 0xc5ce9e3590, 0xc78c466580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6e4649a80, 0xc78c466580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5dec0df80, 0xc8a88ab800, 0x75b, 0x900, 0x2f601e04, 0x280bcc0, 0xc8a88ab800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5dec0df80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 666 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2756f1eb5a9276ff)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc764654660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc764654660, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc764654660, 0x28006b0, 0xc8b1e0c120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc668fcab40, 0xc764654660, 0xc5b7b9ba90, 0xc8b1e0c1d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc668fcab40, 0xc5ce9e35f0, 0xc764654660, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc668fcab60, 0xc764654660, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac0e0, 0xc8cc23d000, 0x768, 0x900, 0x3605f19b, 0x280bcc0, 0xc8cc23d000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 667 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x972c18733766aef0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc801ec78a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc801ec78a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc801ec78a0, 0x28006b0, 0xc5d91790e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7961aa4a0, 0xc801ec78a0, 0xc5aeacfa90, 0xc5d91791a8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7961aa4a0, 0xc5ce9e3650, 0xc801ec78a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7961aa4c0, 0xc801ec78a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac230, 0xc8c647e000, 0x904, 0xc00, 0x34e5b3bd, 0x280bcc0, 0xc8c647e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 668 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x2b598654366bfdb9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8ab6b8c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8ab6b8c60, 0x1a753c0, 0xc5be3b5a18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8ab6b8c60, 0x28006b0, 0xc4fdf77200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7f5323c80, 0xc8ab6b8c60, 0xc5be3b5a90, 0xc4fdf772b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7f5323c80, 0xc5ce9e36b0, 0xc8ab6b8c60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7f5323ca0, 0xc8ab6b8c60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac380, 0xc8baf6c800, 0x79d, 0x900, 0x1ba90e76, 0x280bcc0, 0xc8baf6c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 669 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8c1ca8891e1dfeb3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8806cd1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8806cd1e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8806cd1e0, 0x28006b0, 0xc5d86db7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc80251fc00, 0xc8806cd1e0, 0xc5dceb9a90, 0xc5d86db858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc80251fc00, 0xc5ce9e3710, 0xc8806cd1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc80251fc20, 0xc8806cd1e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac4d0, 0xc8582d7000, 0x786, 0x900, 0x364c3395, 0x280bcc0, 0xc8582d7000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac4d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 670 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x1480c67e1ae024b9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73b4c3aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73b4c3aa0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73b4c3aa0, 0x28006b0, 0xc6f9535d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84d1b6860, 0xc73b4c3aa0, 0xc57eb33a90, 0xc6f9535df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84d1b6860, 0xc5ce9e3770, 0xc73b4c3aa0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84d1b6880, 0xc73b4c3aa0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac620, 0xc83b6a8000, 0x928, 0xc00, 0x162d9148, 0x280bcc0, 0xc83b6a8000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 671 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xde9f7a45f777d2f1)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc66432a400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc66432a400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc66432a400, 0x28006b0, 0xc4b5949b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4af202620, 0xc66432a400, 0xc5af309a90, 0xc4b5949bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4af202620, 0xc5ce9e37d0, 0xc66432a400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4af202680, 0xc66432a400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac770, 0xc97fe96800, 0x7e7, 0x900, 0x7242673, 0x280bcc0, 0xc97fe96800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 672 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x64c9006611f40077)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5e4f3c180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5e4f3c180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5e4f3c180, 0x28006b0, 0xc4f2d7b320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4d2139ae0, 0xc5e4f3c180, 0xc59cb6ba90, 0xc4f2d7b3d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4d2139ae0, 0xc5ce9e3830, 0xc5e4f3c180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4d2139b20, 0xc5e4f3c180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fac8c0, 0xc7d8856000, 0x721, 0x900, 0x36666b0c, 0x280bcc0, 0xc7d8856000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fac8c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 673 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3bb41106cd880b77)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc599f63580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc599f63580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc599f63580, 0x28006b0, 0xc839d5d7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8320e2800, 0xc599f63580, 0xc6373f9a90, 0xc839d5d858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8320e2800, 0xc5ce9e3890, 0xc599f63580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8320e2820, 0xc599f63580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2faca10, 0xc84d672000, 0x95b, 0xc00, 0x27a30fcb, 0x280bcc0, 0xc84d672000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2faca10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 674 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x3efedfec3038f252)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc443422560, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc443422560, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc443422560, 0x28006b0, 0xc86dd330e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc452bb9820, 0xc443422560, 0xc5e1ab7a90, 0xc86dd33198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc452bb9820, 0xc5ce9e38f0, 0xc443422560, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc452bb9840, 0xc443422560, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2facb60, 0xc8829c4000, 0x792, 0x900, 0x22282466, 0x280bcc0, 0xc8829c4000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2facb60, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 675 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xc5d660cd30d301fb)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc81b2801c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc81b2801c0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc81b2801c0, 0x28006b0, 0xc8090846c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7b15279c0, 0xc81b2801c0, 0xc584b99a90, 0xc809084778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7b15279c0, 0xc5ce9e3950, 0xc81b2801c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7b15279e0, 0xc81b2801c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2faccb0, 0xc7abacea00, 0x519, 0x6a0, 0x38940527, 0x280bcc0, 0xc7abacea00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2faccb0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 676 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf13ec284c3f632d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86b8fd320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86b8fd320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86b8fd320, 0x28006b0, 0xc8a99df7a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc84c85cfc0, 0xc86b8fd320, 0xc57fd75a90, 0xc8a99df858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc84c85cfc0, 0xc5ce9e39b0, 0xc86b8fd320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc84c85cfe0, 0xc86b8fd320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2face00, 0xc844d57800, 0x7c9, 0x900, 0x7f0d74, 0x280bcc0, 0xc844d57800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2face00, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 677 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xb2039d7a5e0cfef0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82c0f72c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82c0f72c0, 0x1a753c0, 0xc5fbe7da18)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82c0f72c0, 0x28006b0, 0xc7d2f2fd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8b1c68ca0, 0xc82c0f72c0, 0xc5fbe7da90, 0xc7d2f2fe08)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8b1c68ca0, 0xc5ce9e3a10, 0xc82c0f72c0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8b1c68cc0, 0xc82c0f72c0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2facf50, 0xc987aa1000, 0x861, 0x900, 0xdb7950b, 0x280bcc0, 0xc987aa1000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2facf50, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 678 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206ec184)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206ec180)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x76797186c0632c63)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc614c933e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc614c933e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc614c933e0, 0x28006b0, 0xc7aaacdd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc75d13aa40, 0xc614c933e0, 0xc5b077ba90, 0xc7aaacddf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc75d13aa40, 0xc5ce9e3a70, 0xc614c933e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc75d13aa60, 0xc614c933e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad0a0, 0xc8bb187000, 0x743, 0x900, 0x1fd42c23, 0x280bcc0, 0xc8bb187000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad0a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 679 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf4a8649a787a02b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc816f5ae20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc816f5ae20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc816f5ae20, 0x28006b0, 0xc8895aa5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc699096320, 0xc816f5ae20, 0xc5b70eba90, 0xc8895aa668)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc699096320, 0xc5ce9e3ad0, 0xc816f5ae20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc699096340, 0xc816f5ae20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad1f0, 0xc85fc1e000, 0x6f1, 0x900, 0x13177b47, 0x280bcc0, 0xc85fc1e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad1f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 680 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf855351460c65fa0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86c28eae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86c28eae0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86c28eae0, 0x28006b0, 0xc73ceba6c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc88ad25c60, 0xc86c28eae0, 0xc58658ba90, 0xc73ceba778)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc88ad25c60, 0xc5ce9e3b30, 0xc86c28eae0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc88ad25c80, 0xc86c28eae0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad340, 0xc981b66800, 0x839, 0x900, 0x3a9fc267, 0x280bcc0, 0xc981b66800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad340, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 681 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa3466f50e951753b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7e6be3800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7e6be3800, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7e6be3800, 0x28006b0, 0xc5ea03bd40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc767e4d1c0, 0xc7e6be3800, 0xc79db21a90, 0xc5ea03bdf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc767e4d1c0, 0xc5ce9e3b90, 0xc7e6be3800, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc767e4d1e0, 0xc7e6be3800, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad490, 0xc73025e000, 0x972, 0xc00, 0x37e4a2db, 0x280bcc0, 0xc73025e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad490, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 682 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x9939f8f60e508ec7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc64babb240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc64babb240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc64babb240, 0x28006b0, 0xc8ad4225a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc82f5a1840, 0xc64babb240, 0xc5ab2d7a90, 0xc8ad422658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc82f5a1840, 0xc5ce9e3bf0, 0xc64babb240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc82f5a1860, 0xc64babb240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad5e0, 0xc9880f5000, 0x88f, 0x900, 0x14fd44ff, 0x280bcc0, 0xc9880f5000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad5e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 683 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xaa7ed84abeb715c7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7467325e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7467325e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7467325e0, 0x28006b0, 0xc484341320)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5fe92ecc0, 0xc7467325e0, 0xc5e2fa1a90, 0xc4843413d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5fe92ecc0, 0xc5ce9e3c50, 0xc7467325e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5fe92ece0, 0xc7467325e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad7a0, 0xc708c62000, 0xa66, 0xc00, 0x1ab50ab9, 0x280bcc0, 0xc708c62000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad7a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 684 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x427d34e7f0baec3c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc82b5f6280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc82b5f6280, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc82b5f6280, 0x28006b0, 0xc5864fb8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc703edcda0, 0xc82b5f6280, 0xc5d6505a90, 0xc5864fb978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc703edcda0, 0xc5ce9e3cb0, 0xc82b5f6280, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc703edcdc0, 0xc82b5f6280, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fad8f0, 0xc8bafa6000, 0x927, 0xc00, 0x36e8cce0, 0x280bcc0, 0xc8bafa6000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fad8f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 685 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xd45d7a57208c50a9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7fffa21e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7fffa21e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7fffa21e0, 0x28006b0, 0xc863bb2d80)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8b83b6a00, 0xc7fffa21e0, 0xc57fd93a90, 0xc863bb2e38)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8b83b6a00, 0xc5ce9e3d10, 0xc7fffa21e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8b83b6a20, 0xc7fffa21e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fada40, 0xc8c8f0b800, 0x824, 0x900, 0x3a16367, 0x280bcc0, 0xc8c8f0b800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fada40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 686 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x42de89eabed9de56)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc63e28d400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc63e28d400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc63e28d400, 0x28006b0, 0xc76a618000)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc71ce01da0, 0xc63e28d400, 0xc63146da90, 0xc76a6180b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc71ce01da0, 0xc5ce9e3d70, 0xc63e28d400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc71ce01de0, 0xc63e28d400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fadb90, 0xc82c7da000, 0x68f, 0x6a0, 0x19e3fe1a, 0x280bcc0, 0xc82c7da000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fadb90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 687 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x71dca53c22fd5802)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc654bbc1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc654bbc1e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc654bbc1e0, 0x28006b0, 0xc7cd0470e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc748b46f00, 0xc654bbc1e0, 0xc5b04ffa90, 0xc7cd047198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc748b46f00, 0xc5ce9e3dd0, 0xc654bbc1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc748b46f40, 0xc654bbc1e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fadce0, 0xc986a2a000, 0x87b, 0x900, 0x7b8f2e6, 0x280bcc0, 0xc986a2a000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fadce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 688 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf4abc2e0cda4e536)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7ee929420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7ee929420, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7ee929420, 0x28006b0, 0xc66203cb40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc54849a060, 0xc7ee929420, 0xc5dce59a90, 0xc66203cbf8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc54849a060, 0xc5ce9e3e30, 0xc7ee929420, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc54849a080, 0xc7ee929420, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fade30, 0xc7d2ef2800, 0x72b, 0x900, 0x9836162, 0x280bcc0, 0xc7d2ef2800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fade30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 689 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7fbb95f50d2412a3)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc61e55e1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc61e55e1e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc61e55e1e0, 0x28006b0, 0xc7409aa120)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f2f814e0, 0xc61e55e1e0, 0xc63968ba90, 0xc7409aa1d8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f2f814e0, 0xc5ce9e3e90, 0xc61e55e1e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f2f81500, 0xc61e55e1e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e2fadf80, 0xc97fc39000, 0x6d4, 0x900, 0x1dccf073, 0x280bcc0, 0xc97fc39000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e2fadf80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 690 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x530b40d0732490ce)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7224e8260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7224e8260, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7224e8260, 0x28006b0, 0xc5811dc900)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc608013aa0, 0xc7224e8260, 0xc5b84dba90, 0xc5811dc9b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc608013aa0, 0xc5ce9e3ef0, 0xc7224e8260, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc608013ac0, 0xc7224e8260, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3420e0, 0xc7d02aaa00, 0x620, 0x6a0, 0x999ff9f, 0x280bcc0, 0xc7d02aaa00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3420e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 691 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xaaf4ee7b61fe7046)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc83bed3f60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc83bed3f60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc83bed3f60, 0x28006b0, 0xc6fdb40fc0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc8ab6433e0, 0xc83bed3f60, 0xc5b6991a90, 0xc6fdb41078)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc8ab6433e0, 0xc5ce9e3f50, 0xc83bed3f60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc8ab643440, 0xc83bed3f60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342230, 0xc7f8d98000, 0x6f2, 0x900, 0x23e5267d, 0x280bcc0, 0xc7f8d98000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 692 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf72e4551aef8838e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc74f7bbce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc74f7bbce0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc74f7bbce0, 0x28006b0, 0xc66482f200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc860e3ab20, 0xc74f7bbce0, 0xc88c1bba90, 0xc66482f2b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc860e3ab20, 0xc5ce9e3fb0, 0xc74f7bbce0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc860e3ab40, 0xc74f7bbce0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342380, 0xc7d44cd500, 0x603, 0x6a0, 0xc5297b0, 0x280bcc0, 0xc7d44cd500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 693 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x314d2f013988e76c)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4e3f91a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4e3f91a20, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4e3f91a20, 0x28006b0, 0xc8346a2240)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc788fecc80, 0xc4e3f91a20, 0xc5d99cda90, 0xc8346a22f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc788fecc80, 0xc5ccb78030, 0xc4e3f91a20, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc788fecca0, 0xc4e3f91a20, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3424d0, 0xc7d5f71800, 0x900, 0x900, 0xfd3acb7, 0x280bcc0, 0xc7d5f71800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3424d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 694 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6c1fd703575a7363)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc866d56100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc866d56100, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc866d56100, 0x28006b0, 0xc87636f440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc655fa4560, 0xc866d56100, 0xc5cd3cfa90, 0xc87636f4f8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc655fa4560, 0xc5ccb78090, 0xc866d56100, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc655fa4580, 0xc866d56100, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342620, 0xc7c6f90000, 0x9e0, 0xc00, 0x27c5be3b, 0x280bcc0, 0xc7c6f90000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 695 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d69c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d69c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x724c699932a0031e)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f60979a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f60979a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f60979a0, 0x28006b0, 0xc6a14438c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc45826b3c0, 0xc6f60979a0, 0xc584989a90, 0xc6a1443978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc45826b3c0, 0xc5ccb780f0, 0xc6f60979a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc45826b3e0, 0xc6f60979a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342770, 0xc8582ce000, 0x707, 0x900, 0x3af4df79, 0x280bcc0, 0xc8582ce000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342770, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 696 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xae0847336e5566a5)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc665c07de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc665c07de0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc665c07de0, 0x28006b0, 0xc43e2cd560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4279b4220, 0xc665c07de0, 0xc6052fba90, 0xc43e2cd618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4279b4220, 0xc5ccb78150, 0xc665c07de0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4279b4240, 0xc665c07de0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3428c0, 0xc883433500, 0x672, 0x6a0, 0x24bf47f7, 0x280bcc0, 0xc883433500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3428c0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 697 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x82d1a01264218c52)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7862f3b60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7862f3b60, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7862f3b60, 0x28006b0, 0xc8b5ccb680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5d33230a0, 0xc7862f3b60, 0xc5ad611a90, 0xc8b5ccb738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5d33230a0, 0xc5ccb781b0, 0xc7862f3b60, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5d33230c0, 0xc7862f3b60, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342a10, 0xc88e47c000, 0x933, 0xc00, 0x163121c7, 0x280bcc0, 0xc88e47c000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342a10, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 698 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x8b7d742a493efaed)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc60d442200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc60d442200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc60d442200, 0x28006b0, 0xc8c8d4b8c0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4aa14fc20, 0xc60d442200, 0xc5e3651a90, 0xc8c8d4b978)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4aa14fc20, 0xc5ccb78210, 0xc60d442200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4aa14fc80, 0xc60d442200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342bd0, 0xc7d3899500, 0x561, 0x6a0, 0x1f5de9e4, 0x280bcc0, 0xc7d3899500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342bd0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 699 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4ee51067d47a7727)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc78315b940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc78315b940, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc78315b940, 0x28006b0, 0xc8184f7d40)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5f8eb92e0, 0xc78315b940, 0xc63b733a90, 0xc8184f7df8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5f8eb92e0, 0xc5ccb78270, 0xc78315b940, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5f8eb9340, 0xc78315b940, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342d20, 0xc8876bc000, 0xa21, 0xc00, 0x22dcedb9, 0x280bcc0, 0xc8876bc000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342d20, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 700 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xa82d1afd7dbac6a2)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc556233160, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc556233160, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc556233160, 0x28006b0, 0xc558965440)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc88eca4100, 0xc556233160, 0xc5a5d59a90, 0xc558965508)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc88eca4100, 0xc5ccb782d0, 0xc556233160, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc88eca4120, 0xc556233160, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342e70, 0xc8c6256000, 0x686, 0x6a0, 0x2e4d7502, 0x280bcc0, 0xc8c6256000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342e70, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 701 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d5e44)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d5e40)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x872aba6f0abd872b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc86cb7f040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc86cb7f040, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc86cb7f040, 0x28006b0, 0xc7a7055b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc75ef06e80, 0xc86cb7f040, 0xc5d9059a90, 0xc7a7055bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc75ef06e80, 0xc5ccb78330, 0xc86cb7f040, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc75ef06f20, 0xc86cb7f040, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab342fc0, 0xc7d8579000, 0x750, 0x900, 0x1ebd473e, 0x280bcc0, 0xc7d8579000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab342fc0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 702 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xe262e9c6e0cd36c8)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc886070980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc886070980, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc886070980, 0x28006b0, 0xc58fc23200)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc664b94dc0, 0xc886070980, 0xc602437a90, 0xc58fc232b8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc664b94dc0, 0xc5ccb78390, 0xc886070980, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc664b94de0, 0xc886070980, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343110, 0xc84688c800, 0x86c, 0x900, 0x27450c45, 0x280bcc0, 0xc84688c800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343110, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 703 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf12e2e9c026ba592)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6e17492e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6e17492e0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6e17492e0, 0x28006b0, 0xc8090c50e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6e7700500, 0xc6e17492e0, 0xc5b692ba90, 0xc8090c5198)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6e7700500, 0xc5ccb783f0, 0xc6e17492e0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6e7700520, 0xc6e17492e0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343260, 0xc7d7037800, 0x873, 0x900, 0x2f867c55, 0x280bcc0, 0xc7d7037800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343260, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 704 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xafa891ea811b4890)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc73ef50d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc73ef50d80, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc73ef50d80, 0x28006b0, 0xc84b301b00)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5fa6169a0, 0xc73ef50d80, 0xc584985a90, 0xc84b301bb8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5fa6169a0, 0xc5ccb78450, 0xc73ef50d80, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5fa6169c0, 0xc73ef50d80, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3433b0, 0xc7f9a88800, 0x758, 0x900, 0x1a09e9f8, 0x280bcc0, 0xc7f9a88800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3433b0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 705 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x6bfc3d5ea514e02b)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc7bfdab340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc7bfdab340, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc7bfdab340, 0x28006b0, 0xc72c363e60)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc800880f80, 0xc7bfdab340, 0xc5e2649a90, 0xc72c363f18)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc800880f80, 0xc5ccb784b0, 0xc7bfdab340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc800880fa0, 0xc7bfdab340, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343500, 0xc982af1800, 0x892, 0x900, 0x3420658a, 0x280bcc0, 0xc982af1800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343500, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 706 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x654b22e7234d3e69)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc88e237320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc88e237320, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc88e237320, 0x28006b0, 0xc76cbac5a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6aed252e0, 0xc88e237320, 0xc587f59a90, 0xc76cbac658)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6aed252e0, 0xc5ccb78510, 0xc88e237320, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6aed25340, 0xc88e237320, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343650, 0xc846a56000, 0xa27, 0xc00, 0x2ce8f126, 0x280bcc0, 0xc846a56000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343650, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 707 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x832ca2d42277f399)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc794e2a240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc794e2a240, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc794e2a240, 0x28006b0, 0xc69aae8a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc6fa676380, 0xc794e2a240, 0xc59fe4fa90, 0xc69aae8ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc6fa676380, 0xc5ccb78570, 0xc794e2a240, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc6fa6763a0, 0xc794e2a240, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3437a0, 0xc7b161e000, 0x68f, 0x6a0, 0x1e564472, 0x280bcc0, 0xc7b161e000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3437a0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 708 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x71387f5d449a0a22)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5a1dae580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5a1dae580, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5a1dae580, 0x28006b0, 0xc82cb157a0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc5a6b66100, 0xc5a1dae580, 0xc6064f3a90, 0xc82cb15858)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc5a6b66100, 0xc5ccb785d0, 0xc5a1dae580, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc5a6b66120, 0xc5a1dae580, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab3438f0, 0xc987fe5500, 0x5d6, 0x6a0, 0x97ce62b, 0x280bcc0, 0xc987fe5500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab3438f0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 709 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf23be30b1e0edce0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc869d58900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc869d58900, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc869d58900, 0x28006b0, 0xc5052d8360)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc83f0905c0, 0xc869d58900, 0xc5b5ed5a90, 0xc5052d8418)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc83f0905c0, 0xc5ccb78630, 0xc869d58900, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc83f0905e0, 0xc869d58900, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343a40, 0xc8af734000, 0x6f4, 0x900, 0xa61ecc5, 0x280bcc0, 0xc8af734000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343a40, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 710 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206f50c4)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206f50c0)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x4d9cac39fb80787a)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc60ce11400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc60ce11400, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc60ce11400, 0x28006b0, 0xc8603ee7e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc880dc5d60, 0xc60ce11400, 0xc5d3e71a90, 0xc8603ee898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc880dc5d60, 0xc5ccb78690, 0xc60ce11400, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc880dc5d80, 0xc60ce11400, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343b90, 0xc8aea1f800, 0x8d7, 0x900, 0x253627ff, 0x280bcc0, 0xc8aea1f800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343b90, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 711 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x235a7e9eee6013c4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc5373a9500, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc5373a9500, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc5373a9500, 0x28006b0, 0xc48cb8a480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc570b8eb00, 0xc5373a9500, 0xc5e3477a90, 0xc48cb8a548)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc570b8eb00, 0xc5ccb786f0, 0xc5373a9500, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc570b8eb40, 0xc5373a9500, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343ce0, 0xc8b346f500, 0x605, 0x6a0, 0xe58efd6, 0x280bcc0, 0xc8b346f500)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343ce0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 712 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206c2e84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206c2e80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x65f3ab24d2e40afa)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc77e3e2b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc77e3e2b40, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc77e3e2b40, 0x28006b0, 0xc765f9d9e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4bd9a3100, 0xc77e3e2b40, 0xc6054ada90, 0xc765f9daa8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4bd9a3100, 0xc5ccb78750, 0xc77e3e2b40, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4bd9a3120, 0xc77e3e2b40, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343e30, 0xc7d6c06800, 0x83b, 0x900, 0x86f2c, 0x280bcc0, 0xc7d6c06800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343e30, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 713 [semacquire, 10608 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0xf3a2dcd8a2c7046d)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc4aa2b6340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc4aa2b6340, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc4aa2b6340, 0x28006b0, 0xc7cd8b1680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc4e827d860, 0xc4aa2b6340, 0xc5ce6a3a90, 0xc7cd8b1738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc4e827d860, 0xc5ccb787b0, 0xc4aa2b6340, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc4e827d880, 0xc4aa2b6340, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5ab343f80, 0xc7d9a42000, 0x991, 0xc00, 0x28f0c390, 0x280bcc0, 0xc7d9a42000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5ab343f80, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 714 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x5dfd80958b6bf39f)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc583fc6180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc583fc6180, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc583fc6180, 0x28006b0, 0xc751726a20)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7c5f5c760, 0xc583fc6180, 0xc5dc7d3a90, 0xc751726ad8)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7c5f5c760, 0xc5ccb78810, 0xc583fc6180, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7c5f5c780, 0xc583fc6180, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e0b7e0e0, 0xc8cb3e2800, 0x6e1, 0x900, 0x3605716a, 0x280bcc0, 0xc8cb3e2800)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e0b7e0e0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 715 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206dfc84)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206dfc80)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x332c62b6eb0483c0)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc571026ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc571026ee0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc571026ee0, 0x28006b0, 0xc6932cc480)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc66b70d460, 0xc571026ee0, 0xc5cc9d1a90, 0xc6932cc538)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc66b70d460, 0xc5ccb78870, 0xc571026ee0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc66b70d480, 0xc571026ee0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e0b7e230, 0xc7d5f6d000, 0x7e4, 0x900, 0x2b959188, 0x280bcc0, 0xc7d5f6d000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e0b7e230, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 716 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206bd944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206bd940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x76e95d5a92164ea9)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc6f62159a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc6f62159a0, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc6f62159a0, 0x28006b0, 0xc745393680)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc66fe5fea0, 0xc6f62159a0, 0xc5a4a6fa90, 0xc745393738)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc66fe5fea0, 0xc5ccb788d0, 0xc6f62159a0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc66fe5fec0, 0xc6f62159a0, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e0b7e380, 0xc7f8fcca00, 0x5fb, 0x6a0, 0x4e24f37, 0x280bcc0, 0xc7f8fcca00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e0b7e380, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 717 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206d7844)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206d7840)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x821ba05f199a31a4)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc8b2bced00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc8b2bced00, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc8b2bced00, 0x28006b0, 0xc663ef3560)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc876b6dae0, 0xc8b2bced00, 0xc5b880ba90, 0xc663ef3618)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc876b6dae0, 0xc5ccb78930, 0xc8b2bced00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc876b6db00, 0xc8b2bced00, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e0b7e4d0, 0xc8c6e1ca00, 0x671, 0x6a0, 0x2a810457, 0x280bcc0, 0xc8c6e1ca00)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e0b7e4d0, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 718 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prometheus/prometheus/storage/local.(*fingerprintLocker).Lock(0xc420644080, 0x7dd747ad911f9fd7)
/go/src/github.com/prometheus/prometheus/storage/local/locker.go:62 +0x66
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Append(0xc4201dc000, 0xc512b3f200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:898 +0x115
github.com/prometheus/prometheus/storage.Fanout.Append(0xc420644f80, 0x2, 0x2, 0xc512b3f200, 0x1a753c0, 0x0)
/go/src/github.com/prometheus/prometheus/storage/storage.go:60 +0x66
github.com/prometheus/prometheus/storage.(*Fanout).Append(0xc4206457a0, 0xc512b3f200, 0x28006b0, 0xc813b607e0)
<autogenerated>:3 +0x69
github.com/prometheus/prometheus/retrieval.(*countingAppender).Append(0xc7072086c0, 0xc512b3f200, 0xc5986b5a90, 0xc813b60898)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:261 +0x48
github.com/prometheus/prometheus/retrieval.ruleLabelsAppender.Append(0x27507a0, 0xc7072086c0, 0xc5ccb78990, 0xc512b3f200, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/target.go:201 +0x1af
github.com/prometheus/prometheus/retrieval.(*ruleLabelsAppender).Append(0xc7072086e0, 0xc512b3f200, 0x0, 0x0)
<autogenerated>:35 +0x69
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc5e0b7e620, 0xc857280000, 0x9d3, 0xc00, 0x386ecee7, 0x280bcc0, 0xc857280000)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:519 +0x2ec
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc5e0b7e620, 0xdf8475800, 0x6fc23ac00, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:429 +0x5b8
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:258 +0x3a6
goroutine 719 [semacquire, 10610 minutes]:
sync.runtime_SemacquireMutex(0xc4206db944)
/usr/local/go/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4206db940)
/usr/local/go/src/sync/mutex.go:87 +0x9d
github.com/prom
View raw

(Sorry about that, but we can’t show files that are this big right now.)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment