Skip to content

Instantly share code, notes, and snippets.

@JensRantil
Created March 5, 2017 19:38
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save JensRantil/2d97bba40ebd9512049b9a8a693fbb12 to your computer and use it in GitHub Desktop.
Save JensRantil/2d97bba40ebd9512049b9a8a693fbb12 to your computer and use it in GitHub Desktop.
Stacktrace from when nsqd stops sending heartbeats. See https://github.com/nsqio/nsq/issues/859.
$ ./nsqd -lookupd-tcp-address localhost:4160
[nsqd] 2017/03/05 19:59:58.009064 nsqd v0.3.8 (built w/go1.6.2)
[nsqd] 2017/03/05 19:59:58.009214 ID: 219
[nsqd] 2017/03/05 19:59:58.010869 TOPIC(testtopic): created
[nsqd] 2017/03/05 19:59:58.011079 NSQ: persisting topic/channel metadata to nsqd.219.dat
[nsqd] 2017/03/05 19:59:58.011626 DISKQUEUE(testtopic): readOne() opened testtopic.diskqueue.000000.dat
[nsqd] 2017/03/05 19:59:58.013017 TCP: listening on [::]:4150
[nsqd] 2017/03/05 19:59:58.013208 HTTP: listening on [::]:4151
[nsqd] 2017/03/05 19:59:58.013216 LOOKUP(localhost:4160): adding peer
[nsqd] 2017/03/05 19:59:58.013269 LOOKUP connecting to localhost:4160
[nsqd] 2017/03/05 19:59:58.014634 LOOKUPD(localhost:4160): peer info {TCPPort:4160 HTTPPort:4161 Version:0.3.8 BroadcastAddress:}
[nsqd] 2017/03/05 19:59:58.014688 LOOKUPD(localhost:4160): topic REGISTER testtopic
[nsqd] 2017/03/05 19:59:58.014949 LOOKUPD(localhost:4160): REGISTER testtopic
[nsqd] 2017/03/05 20:08:09.369960 LOOKUPD(localhost:4160): sending heartbeat
[nsqd] 2017/03/05 20:08:13.014159 LOOKUPD(localhost:4160): sending heartbeat
SIGQUIT: quit
PC=0x6355b m=0
goroutine 0 [idle]:
runtime.mach_semaphore_wait(0xc800000803, 0xc800000000, 0x1, 0x100000000033e01, 0xc82014e480, 0x82e400, 0x56d39, 0xffffffffffffffff, 0x100000000, 0x7fff5fbff65c, ...)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/sys_darwin_amd64.s:411 +0xb
runtime.semasleep1(0xffffffffffffffff, 0x100000000)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/os1_darwin.go:423 +0xdf
runtime.semasleep.func1()
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/os1_darwin.go:439 +0x29
runtime.systemstack(0x7fff5fbff660)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/asm_amd64.s:307 +0xab
runtime.semasleep(0xffffffffffffffff, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/os1_darwin.go:440 +0x36
runtime.notesleep(0x82ecc8)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/lock_sema.go:166 +0xed
runtime.stopm()
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/proc.go:1538 +0x10b
runtime.findrunnable(0xc82001aa00, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/proc.go:1976 +0x739
runtime.schedule()
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/proc.go:2075 +0x24f
runtime.park_m(0xc82014e480)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/proc.go:2140 +0x18b
runtime.mcall(0x7fff5fbff7e0)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/asm_amd64.s:233 +0x5b
goroutine 1 [chan receive]:
github.com/judwhite/go-svc/svc.Run(0x92d808, 0xc820028060, 0xc8200b0aa0, 0x2, 0x2, 0x0, 0x0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/.godeps/src/github.com/judwhite/go-svc/svc/svc_other.go:30 +0x2d3
main.main()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/apps/nsqd/nsqd.go:181 +0x1d5
goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/sigqueue.go:116 +0x132
os/signal.loop()
/usr/local/Cellar/go/1.6.2/libexec/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/usr/local/Cellar/go/1.6.2/libexec/src/os/signal/signal_unix.go:28 +0x37
goroutine 33 [IO wait]:
net.runtime_pollWait(0x969fe8, 0x72, 0x929168)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc82018c060, 0x72, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc82018c060, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc82018c000, 0x0, 0x96a198, 0xc82017e900)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_unix.go:426 +0x27c
net.(*TCPListener).AcceptTCP(0xc82017a088, 0xc820178000, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc82017a088, 0x0, 0x0, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/tcpsock_posix.go:264 +0x3d
github.com/nsqio/nsq/internal/protocol.TCPServer(0x969068, 0xc82017a088, 0x96a0f8, 0xc82017a090, 0x96a0a8, 0xc820010a00)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/protocol/tcp_server.go:20 +0x18f
github.com/nsqio/nsq/nsqd.(*NSQD).Main.func1()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:226 +0xf2
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc82017e1a0, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 7 [select]:
github.com/nsqio/nsq/nsqd.(*diskQueue).ioLoop(0xc820001680)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/diskqueue.go:606 +0x71b
created by github.com/nsqio/nsq/nsqd.newDiskQueue
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/diskqueue.go:97 +0x449
goroutine 8 [select]:
github.com/nsqio/nsq/nsqd.(*Topic).messagePump(0xc820079680)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/topic.go:230 +0xda6
github.com/nsqio/nsq/nsqd.NewTopic.func1()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/topic.go:66 +0x20
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc8201104c0, 0xc8200796e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 34 [IO wait]:
net.runtime_pollWait(0x969f28, 0x72, 0x929168)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/netpoll.go:160 +0x60
net.(*pollDesc).Wait(0xc82018c0d0, 0x72, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc82018c0d0, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc82018c070, 0x0, 0x96a198, 0xc8201b4080)
/usr/local/Cellar/go/1.6.2/libexec/src/net/fd_unix.go:426 +0x27c
net.(*TCPListener).AcceptTCP(0xc82017a098, 0x1, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc82017a098, 0x0, 0x0, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/tcpsock_posix.go:264 +0x3d
net/http.(*Server).Serve(0xc8201ba000, 0x969068, 0xc82017a098, 0x0, 0x0)
/usr/local/Cellar/go/1.6.2/libexec/src/net/http/server.go:2117 +0x129
github.com/nsqio/nsq/internal/http_api.Serve(0x969068, 0xc82017a098, 0x9e9000, 0xc82017e260, 0x58d4b8, 0x4, 0x96a0a8, 0xc820010a00)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/http_api/http_server.go:29 +0x3d7
github.com/nsqio/nsq/nsqd.(*NSQD).Main.func3()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:253 +0xfe
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc82017e8c0, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 35 [select]:
github.com/nsqio/nsq/nsqd.(*NSQD).queueScanLoop(0xc8200b2700)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:667 +0x564
github.com/nsqio/nsq/nsqd.(*NSQD).Main.func4()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:256 +0x20
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc820170cb0, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 36 [select]:
github.com/nsqio/nsq/nsqd.(*NSQD).idPump(0xc8200b2700)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:543 +0x40d
github.com/nsqio/nsq/nsqd.(*NSQD).Main.func5()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:257 +0x20
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc820170cc0, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 37 [select]:
github.com/nsqio/nsq/nsqd.(*NSQD).lookupLoop(0xc8200b2700)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/lookup.go:80 +0x3165
github.com/nsqio/nsq/nsqd.(*NSQD).Main.func6()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:258 +0x20
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc820170cd0, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
goroutine 38 [select, locked to thread]:
runtime.gopark(0x67ead0, 0xc820021f28, 0x593980, 0x6, 0x18, 0x2)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/proc.go:262 +0x163
runtime.selectgoImpl(0xc820021f28, 0x0, 0x18)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/select.go:392 +0xa67
runtime.selectgo(0xc820021f28)
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/select.go:215 +0x12
runtime.ensureSigM.func1()
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/signal1_unix.go:279 +0x32c
runtime.goexit()
/usr/local/Cellar/go/1.6.2/libexec/src/runtime/asm_amd64.s:1998 +0x1
goroutine 18 [select]:
github.com/nsqio/nsq/nsqd.(*NSQD).queueScanWorker(0xc8200b2700, 0xc8201440c0, 0xc820186080, 0xc820144120)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:625 +0x1a7
github.com/nsqio/nsq/nsqd.(*NSQD).resizePool.func1()
/Users/mreiferson/dev/src/github.com/nsqio/nsq/nsqd/nsqd.go:614 +0x3b
github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap.func1(0xc82019e000, 0xc8200b27e0)
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:14 +0x1d
created by github.com/nsqio/nsq/internal/util.(*WaitGroupWrapper).Wrap
/Users/mreiferson/dev/src/github.com/nsqio/nsq/internal/util/wait_group_wrapper.go:16 +0x5d
rax 0xe
rbx 0x82ebc0
rcx 0x7fff5fbff5e8
rdx 0x7fff5fbff660
rdi 0x803
rsi 0x11
rbp 0x803
rsp 0x7fff5fbff5e8
r8 0x82ebc0
r9 0xc820056660
r10 0x3
r11 0x286
r12 0xb681b2dc9a7
r13 0xcb6e951e6c0
r14 0x14a911f49577ac00
r15 0x82de00
rip 0x6355b
rflags 0x286
cs 0x7
fs 0x0
gs 0x0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment