nomad-autoscaler SIGABRT

gistfile1.txt

2022-06-29T08:49:57.214Z [ERROR] policy_manager.policy_handler: timeout: failed to read policy in time: policy_id=1a9ab501-245a-f40a-9904-40a8f2c7dce2
SIGABRT: abort
PC=0x46aac1 m=0 sigcode=0

goroutine 0 [idle]:
runtime.futex()
	/usr/local/go/src/runtime/sys_linux_amd64.s:519 +0x21
runtime.futexsleep(0x13a, 0x2561be0, 0xc00004c800)
	/usr/local/go/src/runtime/os_linux.go:44 +0x36
runtime.notesleep(0x2561d30)
	/usr/local/go/src/runtime/lock_futex.go:160 +0x87
runtime.mPark()
	/usr/local/go/src/runtime/proc.go:1441 +0x2a
runtime.stoplockedm()
	/usr/local/go/src/runtime/proc.go:2602 +0x65
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:3299 +0x3d
runtime.park_m(0xc000484340)
	/usr/local/go/src/runtime/proc.go:3516 +0x14d
runtime.mcall()
	/usr/local/go/src/runtime/asm_amd64.s:307 +0x43

goroutine 1 [chan receive, 8063 minutes]:
github.com/hashicorp/nomad-autoscaler/agent.(*Agent).handleSignals(0xc0004581c0)
	/home/circleci/project/project/agent/agent.go:270 +0xe5
github.com/hashicorp/nomad-autoscaler/agent.(*Agent).Run(0xc0004581c0)
	/home/circleci/project/project/agent/agent.go:100 +0x446
github.com/hashicorp/nomad-autoscaler/command.(*AgentCommand).Run(0xc000428810, {0xc0000c4020, 0x203000, 0x155})
	/home/circleci/project/project/command/agent.go:333 +0x8e5
github.com/mitchellh/cli.(*CLI).Run(0xc000411900)
	/go/pkg/mod/github.com/mitchellh/cli@v1.1.2/cli.go:262 +0x5f8
main.main()
	/home/circleci/project/project/main.go:26 +0x24d

goroutine 5 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000200400)
	/go/pkg/mod/go.opencensus.io@v0.22.5/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
	/go/pkg/mod/go.opencensus.io@v0.22.5/stats/view/worker.go:34 +0x92

goroutine 7 [IO wait]:
internal/poll.runtime_pollWait(0x7f20a4610258, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc000201080, 0x416f46, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000201080)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000201080)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000d440)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000d440)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc00026c9a0, {0x1a5bd18, 0xc00000d440})
	/usr/local/go/src/net/http/server.go:3002 +0x394
github.com/hashicorp/nomad-autoscaler/agent/http.(*Server).Start(0xc000207ae0)
	/home/circleci/project/project/agent/http/server.go:124 +0xd2
created by github.com/hashicorp/nomad-autoscaler/command.(*AgentCommand).Run
	/home/circleci/project/project/command/agent.go:330 +0x896

goroutine 20 [chan receive]:
github.com/hashicorp/nomad-autoscaler/plugins/builtin/target/nomad/plugin.(*TargetPlugin).garbageCollectionLoop(0xc0000c0c00)
	/home/circleci/project/project/plugins/builtin/target/nomad/plugin/plugin.go:213 +0xc9
created by github.com/hashicorp/nomad-autoscaler/plugins/builtin/target/nomad/plugin.(*TargetPlugin).SetConfig
	/home/circleci/project/project/plugins/builtin/target/nomad/plugin/plugin.go:94 +0xef

goroutine 24 [syscall, 8063 minutes]:
os/signal.signal_recv()
	/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x2c

goroutine 25 [select, 8078 minutes]:
github.com/armon/go-metrics.(*InmemSignal).run(0xc0001aa600)
	/go/pkg/mod/github.com/armon/go-metrics@v0.3.7/inmem_signal.go:64 +0x6c
created by github.com/armon/go-metrics.NewInmemSignal
	/go/pkg/mod/github.com/armon/go-metrics@v0.3.7/inmem_signal.go:38 +0x174

goroutine 27 [sleep]:
time.Sleep(0x3b9aca00)
	/usr/local/go/src/runtime/time.go:193 +0x12e
github.com/armon/go-metrics.(*Metrics).collectStats(0xc00010a0f0)
	/go/pkg/mod/github.com/armon/go-metrics@v0.3.7/metrics.go:230 +0x25
created by github.com/armon/go-metrics.New
	/go/pkg/mod/github.com/armon/go-metrics@v0.3.7/start.go:84 +0x17b

goroutine 2160 [select]:
github.com/hashicorp/nomad-autoscaler/policy.(*Manager).periodicMetricsReporter(0xc00009d590, {0x1a6fb30, 0xc000534e00}, 0xc0000c15c0)
	/home/circleci/project/project/policy/manager.go:224 +0xdf
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:61 +0x196

goroutine 29 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d630, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 30 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d680, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 31 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d6d0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 32 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d720, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 33 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d770, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 34 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d7c0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 35 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d810, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 36 [select, 1 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d860, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 37 [select, 1 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d8b0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 38 [select]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x1843df3, 0xa})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d900, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:128 +0x25f

goroutine 39 [select, 8073 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d950, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 40 [select, 8072 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d9a0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 41 [select, 8072 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009d9f0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 42 [select, 8071 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009da40, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 43 [select, 8071 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009da90, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 44 [select, 8070 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009dae0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 45 [select, 8070 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009db30, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 46 [select, 8069 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009db80, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 47 [select, 8069 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009dbd0, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 48 [select, 8068 minutes]:
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).waitForWork(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:246 +0x1e5
github.com/hashicorp/nomad-autoscaler/policyeval.(*Broker).Dequeue(0xc0000c1620, {0x1a6fb30, 0xc0004803c0}, {0x183fb5a, 0x7})
	/home/circleci/project/project/policyeval/broker.go:176 +0x19a
github.com/hashicorp/nomad-autoscaler/policyeval.(*BaseWorker).Run(0xc00009dc20, {0x1a6fb30, 0xc0004803c0})
	/home/circleci/project/project/policyeval/base_worker.go:60 +0xb6
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).initWorkers
	/home/circleci/project/project/agent/agent.go:134 +0x35b

goroutine 49 [select]:
github.com/hashicorp/nomad-autoscaler/agent.(*Agent).runEvalHandler(0xc0004581c0, {0x1a6fb30, 0xc0004803c0}, 0xc0000c15c0)
	/home/circleci/project/project/agent/agent.go:106 +0xb2
created by github.com/hashicorp/nomad-autoscaler/agent.(*Agent).Run
	/home/circleci/project/project/agent/agent.go:97 +0x439

goroutine 2177 [select, 8063 minutes]:
github.com/hashicorp/nomad-autoscaler/policy/file.(*Source).MonitorIDs(0xc000122f80, {0x1a6fb30, 0xc000534e00}, {0xc00057b140, 0xc00057b0e0})
	/home/circleci/project/project/policy/file/source.go:84 +0x131
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:67 +0x1f3

goroutine 2178 [select, 4 minutes]:
github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorIDs(0xc0001aa6c0, {0x1a6fb30, 0xc000534e00}, {0xc00057b140, 0xc00057b0e0})
	/home/circleci/project/project/policy/nomad/source.go:113 +0x26d
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:67 +0x1f3

goroutine 2159 [select, 8063 minutes]:
github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run(0xc00009d590, {0x1a6fb30, 0xc0004803c0}, 0xc0000c15c0)
	/home/circleci/project/project/policy/manager.go:72 +0x475
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:154 +0x847

goroutine 2168 [select, 8063 minutes]:
github.com/hashicorp/nomad-autoscaler/policy/file.(*Source).MonitorPolicy(0xc000122f80, {0x1a6fb30, 0xc000721680}, {{0xc0000bc900, 0x24}, 0xc0002aca80, 0xc0002acba0, 0xc0002ac360})
	/home/circleci/project/project/policy/file/source.go:149 +0x7bc
created by github.com/hashicorp/nomad-autoscaler/policy.(*Handler).Run
	/home/circleci/project/project/policy/handler.go:115 +0x2e5

goroutine 76 [select]:
net/http.(*persistConn).roundTrip(0xc00021e120, 0xc0007a0240)
	/usr/local/go/src/net/http/transport.go:2614 +0x97d
net/http.(*Transport).roundTrip(0xc000264dc0, 0xc00010d500)
	/usr/local/go/src/net/http/transport.go:594 +0x7d1
net/http.(*Transport).RoundTrip(0xc000581960, 0x1a3c1e0)
	/usr/local/go/src/net/http/roundtrip.go:18 +0x19
net/http.send(0xc00010d500, {0x1a3c1e0, 0xc000264dc0}, {0x17ff560, 0x40ce01, 0x0})
	/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc00061d2c0, 0xc00010d500, {0x203000, 0x203000, 0x0})
	/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc00061d2c0, 0xc00010d500)
	/usr/local/go/src/net/http/client.go:725 +0x908
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:593
github.com/hashicorp/nomad/api.(*Client).doRequest(0xc000434620, 0xc000581f30)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:722 +0x65
github.com/hashicorp/nomad/api.(*Client).query(0x184a270, {0xc00055e880, 0xc000581ea8}, {0x148d380, 0xc0007a01c0}, 0xc00050f4b8)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:841 +0xb4
github.com/hashicorp/nomad/api.(*Jobs).ScaleStatus(0xc000581f28, {0xc0004ce4a0, 0x10}, 0x0)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/jobs.go:190 +0xb8
github.com/hashicorp/nomad-autoscaler/plugins/builtin/target/nomad/plugin.(*jobScaleStatusHandler).start(0xc00050f480)
	/home/circleci/project/project/plugins/builtin/target/nomad/plugin/state.go:165 +0x107
created by github.com/hashicorp/nomad-autoscaler/plugins/builtin/target/nomad/plugin.newJobScaleStatusHandler
	/home/circleci/project/project/plugins/builtin/target/nomad/plugin/state.go:74 +0x22f

goroutine 1234495 [select, 4 minutes]:
net/http.(*persistConn).roundTrip(0xc000022120, 0xc0001aa380)
	/usr/local/go/src/net/http/transport.go:2614 +0x97d
net/http.(*Transport).roundTrip(0xc000264b40, 0xc00069c100)
	/usr/local/go/src/net/http/transport.go:594 +0x7d1
net/http.(*Transport).RoundTrip(0x30, 0x1a3c1e0)
	/usr/local/go/src/net/http/roundtrip.go:18 +0x19
net/http.send(0xc00069c100, {0x1a3c1e0, 0xc000264b40}, {0x17ff560, 0x40ce01, 0x0})
	/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc0006e0ed0, 0xc00069c100, {0x203000, 0x203000, 0x0})
	/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc0006e0ed0, 0xc00069c100)
	/usr/local/go/src/net/http/client.go:725 +0x908
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:593
github.com/hashicorp/nomad/api.(*Client).doRequest(0xc0004340e0, 0xc000486a20)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:722 +0x65
github.com/hashicorp/nomad/api.(*Client).query(0xc000122580, {0x184fced, 0x1b899119c034d}, {0x1478d40, 0xc00050c078}, 0x1247200)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:841 +0xb4
github.com/hashicorp/nomad/api.(*Scaling).ListPolicies(0xc000505fb0, 0xc000473484)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/scaling.go:20 +0x65
github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorIDs.func1()
	/home/circleci/project/project/policy/nomad/source.go:109 +0xca
created by github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorIDs
	/home/circleci/project/project/policy/nomad/source.go:102 +0x1e6

goroutine 2167 [select, 1 minutes]:
github.com/hashicorp/nomad-autoscaler/policy.(*Handler).Run(0xc0006cd5e0, {0x1a6fb30, 0xc0004803c0}, 0x0)
	/home/circleci/project/project/policy/handler.go:118 +0x411
github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run.func1({0xc0000bc900, 0x24})
	/home/circleci/project/project/policy/manager.go:116 +0x46
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:115 +0xed3

goroutine 2186 [select]:
github.com/hashicorp/nomad-autoscaler/policy.(*Handler).Run(0xc00047c1e0, {0x1a6fb30, 0xc0004803c0}, 0x0)
	/home/circleci/project/project/policy/handler.go:118 +0x411
github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run.func1({0xc000492bd0, 0x24})
	/home/circleci/project/project/policy/manager.go:116 +0x46
created by github.com/hashicorp/nomad-autoscaler/policy.(*Manager).Run
	/home/circleci/project/project/policy/manager.go:115 +0xed3

goroutine 1234531 [select, 4 minutes]:
net/http.(*persistConn).writeLoop(0xc000022120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 2187 [select, 5 minutes]:
github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorPolicy(0xc0001aa6c0, {0x1a6fb30, 0xc0007a0140}, {{0xc000492bd0, 0x24}, 0xc0004c02a0, 0xc0004c03c0, 0xc0004c0240})
	/home/circleci/project/project/policy/nomad/source.go:202 +0x3ed
created by github.com/hashicorp/nomad-autoscaler/policy.(*Handler).Run
	/home/circleci/project/project/policy/handler.go:115 +0x2e5

goroutine 1235052 [IO wait]:
internal/poll.runtime_pollWait(0x7f20a460fa30, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00046e000, 0xc00063d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00046e000, {0xc00063d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00046e000, {0xc00063d000, 0x4cee66, 0x2560c80})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e008, {0xc00063d000, 0x0, 0xc00007c428})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).Read(0xc00007c420, {0xc00063d000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:780 +0x16d
bufio.(*Reader).fill(0xc0007640c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0007640c0, 0x4)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*conn).serve(0xc0002ae0a0, {0x1a6fbd8, 0xc00018d320})
	/usr/local/go/src/net/http/server.go:1955 +0xc36
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3034 +0x4e8

goroutine 1234368 [select, 5 minutes]:
net/http.(*persistConn).writeLoop(0xc00076a6c0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1234530 [IO wait, 4 minutes]:
internal/poll.runtime_pollWait(0x7f20a460fce8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00068a080, 0xc00057d000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00068a080, {0xc00057d000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00068a080, {0xc00057d000, 0x11, 0xc000730c70})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000be0d8, {0xc00057d000, 0x11, 0xc000730cf0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc000022120, {0xc00057d000, 0x40505d, 0x60})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc00057a7e0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc00057a7e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc000022120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1235107 [select]:
net/http.(*persistConn).writeLoop(0xc00021e120)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1235106 [IO wait]:
internal/poll.runtime_pollWait(0x7f20a4610088, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00050f080, 0xc00002f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00050f080, {0xc00002f000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00050f080, {0xc00002f000, 0xc74e1f, 0x2})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000be9d8, {0xc00002f000, 0x0, 0xc0005b8cc8})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00021e120, {0xc00002f000, 0x40505d, 0x60})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc000698300)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc000698300, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00021e120)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1234367 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f20a460f778, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00050e480, 0xc00048b000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00050e480, {0xc00048b000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00050e480, {0xc00048b000, 0x546ae9, 0x0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e7a0, {0xc00048b000, 0x18, 0xc0003884d8})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc00076a6c0, {0xc00048b000, 0x40505d, 0x60})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0004cc360)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0004cc360, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc00076a6c0)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1234364 [select, 5 minutes]:
net/http.(*persistConn).roundTrip(0xc00076a6c0, 0xc000530100)
	/usr/local/go/src/net/http/transport.go:2614 +0x97d
net/http.(*Transport).roundTrip(0xc000264b40, 0xc000748000)
	/usr/local/go/src/net/http/transport.go:594 +0x7d1
net/http.(*Transport).RoundTrip(0x30, 0x1a3c1e0)
	/usr/local/go/src/net/http/roundtrip.go:18 +0x19
net/http.send(0xc000748000, {0x1a3c1e0, 0xc000264b40}, {0x17ff560, 0x40ce01, 0x0})
	/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc0006e0ed0, 0xc000748000, {0x203000, 0x203000, 0x0})
	/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc0006e0ed0, 0xc000748000)
	/usr/local/go/src/net/http/client.go:725 +0x908
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:593
github.com/hashicorp/nomad/api.(*Client).doRequest(0xc0004340e0, 0xc00027c120)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:722 +0x65
github.com/hashicorp/nomad/api.(*Client).query(0x1247094, {0xc0003c8580, 0x1243782}, {0x1595120, 0xc000408070}, 0x1b8832ec5d74d)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/api.go:841 +0xb4
github.com/hashicorp/nomad/api.(*Scaling).GetPolicy(0xc0003887b0, {0xc000492bd0, 0x24}, 0xc0000c6364)
	/go/pkg/mod/github.com/hashicorp/nomad/api@v0.0.0-20210503143957-4ccada7924cf/scaling.go:29 +0x88
github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorPolicy.func1()
	/home/circleci/project/project/policy/nomad/source.go:198 +0xe9
created by github.com/hashicorp/nomad-autoscaler/policy/nomad.(*Source).MonitorPolicy
	/home/circleci/project/project/policy/nomad/source.go:191 +0x367

goroutine 1234989 [IO wait]:
internal/poll.runtime_pollWait(0x7f20a460fdd0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00046f100, 0xc00046c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00046f100, {0xc00046c000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00046f100, {0xc00046c000, 0x438127, 0xc00072ec30})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000beec8, {0xc00046c000, 0x11, 0xc0003e5ba0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc000022c60, {0xc00046c000, 0xc00069f5c0, 0xc00072ed30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc000510480)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc000510480, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc000022c60)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1234990 [select]:
net/http.(*persistConn).writeLoop(0xc000022c60)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

rax    0xca
rbx    0x0
rcx    0x46aac3
rdx    0x0
rdi    0x2561d30
rsi    0x80
rbp    0x7ffde92b7b00
rsp    0x7ffde92b7ab8
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0x43e700
r13    0x0
r14    0x2561400
r15    0x0
rip    0x46aac1
rflags 0x286
cs     0x33
fs     0x0
gs     0x0
2022-06-29T08:51:13.473Z [INFO]  agent: Starting Nomad Autoscaler agent