Stacktrace for influxdb issue #1003

influxdb#1033

[10/16/14 07:35:13] [INFO] Loading configuration file /opt/influxdb/shared/config.toml
[10/16/14 07:35:13] [INFO] Redirectoring logging to stdout
[10/16/14 07:35:13] [INFO] Starting Influx Server 0.8.3 bound to 0.0.0.0...
[10/16/14 07:35:13] [INFO] Opening database at /var/lib/influxdb/db
[10/16/14 07:35:13] [INFO] Opening wal in /var/lib/influxdb/wal
[10/16/14 07:35:13] [INFO] Ssl will be disabled since the ssl port or certificate path weren't set
[10/16/14 07:35:13] [INFO] Initializing Raft HTTP server
[10/16/14 07:35:13] [INFO] Raft Server Listening at 0.0.0.0:8090
[10/16/14 07:35:13] [INFO] Initializing Raft Server: http://influx1e:8090
[10/16/14 07:35:13] [INFO] (raft:67e282ae6e3c28d1) Attempting to join leader: influx1b:8090
[10/16/14 07:35:13] [INFO] Recovering the cluster configuration
[10/16/14 07:35:13] [INFO] Checking whether dadedffc55a1a008 is the local server 67e282ae6e3c28d1
[10/16/14 07:35:13] [INFO] server: 1: Initializing write buffer with buffer size of 1000
[10/16/14 07:35:13] [INFO] ClusterServer: 1 connecting to: influx1b:8099
panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x0 pc=0x66e749]

goroutine 25 [running]:
runtime.panic(0xbfc980, 0x120f153)
        /.gvm/gos/go1.3.1/src/pkg/runtime/panic.c:279 +0xf5
github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).convertNewShardDataToShards(0xc2080c2000, 0xc2084241f8, 0x1, 0x1, 0x0, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:633 +0x259
github.com/influxdb/influxdb/cluster.(*ClusterConfiguration).Recovery(0xc2080c2000, 0xc208594000, 0x1312ff, 0x132000, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_configuration.go:689 +0x7d2
github.com/influxdb/influxdb/_vendor/raft.(*server).processSnapshotRecoveryRequest(0xc208003200, 0xc2080c0280, 0xc2080c0280)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:1291 +0x7c
github.com/influxdb/influxdb/_vendor/raft.(*server).snapshotLoop(0xc208003200)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:881 +0x27e
github.com/influxdb/influxdb/_vendor/raft.(*server).loop(0xc208003200)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:611 +0x425
github.com/influxdb/influxdb/_vendor/raft.func·007()
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:470 +0x5d
created by github.com/influxdb/influxdb/_vendor/raft.(*server).Start
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:471 +0x3b8

goroutine 16 [chan receive]:
github.com/influxdb/influxdb/coordinator.(*RaftServer).Serve(0xc208072000, 0x7f07a6412628, 0xc208030030, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:587 +0x4f5
github.com/influxdb/influxdb/coordinator.(*RaftServer).ListenAndServe(0xc208072000, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:559 +0xed
github.com/influxdb/influxdb/server.(*Server).ListenAndServe(0xc20803a090, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/server/server.go:86 +0x50
main.main()
        /gocodez/src/github.com/influxdb/influxdb/daemon/influxd.go:166 +0xec7

goroutine 19 [finalizer wait]:
runtime.park(0x4f7bc0, 0x1227900, 0x1211ec9)
        /.gvm/gos/go1.3.1/src/pkg/runtime/proc.c:1369 +0x89
runtime.parkunlock(0x1227900, 0x1211ec9)
        /.gvm/gos/go1.3.1/src/pkg/runtime/proc.c:1385 +0x3b
runfinq()
        /.gvm/gos/go1.3.1/src/pkg/runtime/mgc0.c:2644 +0xcf
runtime.goexit()
        /.gvm/gos/go1.3.1/src/pkg/runtime/proc.c:1445

goroutine 20 [syscall]:
os/signal.loop()
        /.gvm/gos/go1.3.1/src/pkg/os/signal/signal_unix.go:21 +0x1e
created by os/signal.init·1
        /.gvm/gos/go1.3.1/src/pkg/os/signal/signal_unix.go:27 +0x32

goroutine 21 [chan receive]:
code.google.com/p/log4go.ConsoleLogWriter.run(0xc208070000, 0x7f07a6410440, 0xc208030008)
        /gocodez/src/code.google.com/p/log4go/termlog.go:27 +0x79
created by code.google.com/p/log4go.NewConsoleLogWriter
        /gocodez/src/code.google.com/p/log4go/termlog.go:19 +0x68

goroutine 17 [syscall]:
runtime.goexit()
        /.gvm/gos/go1.3.1/src/pkg/runtime/proc.c:1445

goroutine 22 [runnable]:
code.google.com/p/log4go.ConsoleLogWriter.run(0xc208070420, 0x7f07a6410440, 0xc208030008)
        /gocodez/src/code.google.com/p/log4go/termlog.go:27 +0x79
created by code.google.com/p/log4go.NewConsoleLogWriter
        /gocodez/src/code.google.com/p/log4go/termlog.go:19 +0x68

goroutine 23 [chan receive]:
github.com/influxdb/influxdb/wal.(*WAL).processEntries(0xc20802a000)
        /gocodez/src/github.com/influxdb/influxdb/wal/wal.go:252 +0x64
created by github.com/influxdb/influxdb/wal.NewWAL
        /gocodez/src/github.com/influxdb/influxdb/wal/wal.go:103 +0xa53

goroutine 24 [chan receive]:
main.waitForSignals(0x7f07a64115a8, 0xc20803a090)
        /gocodez/src/github.com/influxdb/influxdb/daemon/null_profiler.go:23 +0x14c
created by main.startProfiler
        /gocodez/src/github.com/influxdb/influxdb/daemon/null_profiler.go:15 +0x4b

goroutine 33 [IO wait]:
net.runtime_pollWait(0x7f07a64135d0, 0x72, 0x0)
        //.gvm/gos/go1.3.1/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc2080ba060, 0x72, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc2080ba060, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).accept(0xc2080ba000, 0xd94c70, 0x0, 0x7f07a64102b8, 0xb)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_unix.go:419 +0x343
net.(*TCPListener).AcceptTCP(0xc208030030, 0x8, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/tcpsock_posix.go:234 +0x5d
net.(*TCPListener).Accept(0xc208030030, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/tcpsock_posix.go:244 +0x4b
net/http.(*Server).Serve(0xc2080b0360, 0x7f07a6412628, 0xc208030030, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1698 +0x91
github.com/influxdb/influxdb/coordinator.func·008()
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:578 +0x3a
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).Serve
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:582 +0x44c

goroutine 34 [select]:
net/http.(*persistConn).roundTrip(0xc2080726e0, 0xc208056f30, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:1015 +0x6db
net/http.(*Transport).RoundTrip(0xc20803c880, 0xc2080d44e0, 0x4, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:208 +0x49a
net/http.send(0xc2080d44e0, 0x7f07a6410728, 0xc20803c880, 0xc208182174, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/client.go:195 +0x43d
net/http.(*Client).send(0xc20816c570, 0xc2080d44e0, 0x2a, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/client.go:118 +0x15b
net/http.(*Client).doFollowingRedirects(0xc20816c570, 0xc2080d44e0, 0xd94ce0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/client.go:343 +0x97f
net/http.(*Client).Post(0xc20816c570, 0xc20816c420, 0x2a, 0xcb75b0, 0x10, 0x7f07a6413a68, 0xc208182070, 0xc2081679a8, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/client.go:417 +0x111
github.com/influxdb/influxdb/coordinator.(*RaftServer).Join(0xc208072000, 0xc20816c420, 0x2a, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:649 +0x6c0
github.com/influxdb/influxdb/coordinator.(*RaftServer).Join(0xc208072000, 0xc208037cc0, 0x1e, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:658 +0x90a
github.com/influxdb/influxdb/coordinator.(*RaftServer).startRaft(0xc208072000, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:430 +0xa6e
github.com/influxdb/influxdb/coordinator.func·009()
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:585 +0x32
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).Serve
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:586 +0x4bd

goroutine 26 [select]:
github.com/influxdb/influxdb/coordinator.(*RaftServer).CompactLog(0xc208072000)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:347 +0x332
created by github.com/influxdb/influxdb/coordinator.(*RaftServer).startRaft
        /gocodez/src/github.com/influxdb/influxdb/coordinator/raft_server.go:401 +0x487

goroutine 38 [select]:
github.com/influxdb/influxdb/_vendor/raft.(*server).send(0xc208003200, 0xbce880, 0xc2080c0280, 0x0, 0x0, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:630 +0x279
github.com/influxdb/influxdb/_vendor/raft.(*server).SnapshotRecoveryRequest(0xc208003200, 0xc2080c0280, 0xc20800fb00)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:1284 +0x45
github.com/influxdb/influxdb/_vendor/raft.func·004(0x7f07a6413f70, 0xc20803e820, 0xc2080289c0)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/http_transporter.go:315 +0x254
net/http.HandlerFunc.ServeHTTP(0xc2080569a0, 0x7f07a6413f70, 0xc20803e820, 0xc2080289c0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1235 +0x40
github.com/gorilla/mux.(*Router).ServeHTTP(0xc2080c01e0, 0x7f07a6413f70, 0xc20803e820, 0xc2080289c0)
        /gocodez/src/github.com/gorilla/mux/mux.go:98 +0x292
net/http.serverHandler.ServeHTTP(0xc2080b0360, 0x7f07a6413f70, 0xc20803e820, 0xc2080289c0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1673 +0x19f
net/http.(*conn).serve(0xc2080c4380)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1174 +0xa7e
created by net/http.(*Server).Serve
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1721 +0x313

goroutine 36 [IO wait]:
net.runtime_pollWait(0x7f07a6413520, 0x72, 0x0)
        //.gvm/gos/go1.3.1/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc2080bafb0, 0x72, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc2080bafb0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc2080baf50, 0xc20817e000, 0x1000, 0x1000, 0x0, 0x7f07a64102b8, 0xb)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_unix.go:242 +0x34c
net.(*conn).Read(0xc208030110, 0xc20817e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/net.go:122 +0xe7
net/http.noteEOFReader.Read(0x7f07a6413c28, 0xc208030110, 0xc20817c058, 0xc20817e000, 0x1000, 0x1000, 0x4e8ea0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:1203 +0x72
net/http.(*noteEOFReader).Read(0xc2080378c0, 0xc20817e000, 0x1000, 0x1000, 0xc20805a0e0, 0x0, 0x0)
        <autogenerated>:124 +0xca
bufio.(*Reader).fill(0xc2080b0660)
        /.gvm/gos/go1.3.1/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).Peek(0xc2080b0660, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/bufio/bufio.go:132 +0x101
net/http.(*persistConn).readLoop(0xc20817c000)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:782 +0x95
created by net/http.(*Transport).dialConn
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:600 +0x93f

goroutine 37 [select]:
net/http.(*persistConn).writeLoop(0xc20817c000)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:885 +0x38f
created by net/http.(*Transport).dialConn
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:601 +0x957

goroutine 31 [select]:
github.com/influxdb/influxdb/_vendor/raft.(*server).send(0xc208003200, 0xbcea40, 0xc208a560c0, 0x0, 0x0, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:630 +0x279
github.com/influxdb/influxdb/_vendor/raft.(*server).RequestSnapshot(0xc208003200, 0xc208a560c0, 0xc208a6a000)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/server.go:1262 +0x45
github.com/influxdb/influxdb/_vendor/raft.func·003(0x7f07a6413f70, 0xc20803e0a0, 0xc2080d4750)
        /gocodez/src/github.com/influxdb/influxdb/_vendor/raft/http_transporter.go:292 +0x254
net/http.HandlerFunc.ServeHTTP(0xc2080567d0, 0x7f07a6413f70, 0xc20803e0a0, 0xc2080d4750)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1235 +0x40
github.com/gorilla/mux.(*Router).ServeHTTP(0xc2080c01e0, 0x7f07a6413f70, 0xc20803e0a0, 0xc2080d4750)
        /gocodez/src/github.com/gorilla/mux/mux.go:98 +0x292
net/http.serverHandler.ServeHTTP(0xc2080b0360, 0x7f07a6413f70, 0xc20803e0a0, 0xc2080d4750)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1673 +0x19f
net/http.(*conn).serve(0xc20803c900)
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1174 +0xa7e
created by net/http.(*Server).Serve
        /.gvm/gos/go1.3.1/src/pkg/net/http/server.go:1721 +0x313

goroutine 29 [IO wait]:
net.runtime_pollWait(0x7f07a6413470, 0x72, 0x0)
        //.gvm/gos/go1.3.1/src/pkg/runtime/netpoll.goc:146 +0x66
net.(*pollDesc).Wait(0xc208182220, 0x72, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:84 +0x46
net.(*pollDesc).WaitRead(0xc208182220, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_poll_runtime.go:89 +0x42
net.(*netFD).Read(0xc2081821c0, 0xc20808b000, 0x1000, 0x1000, 0x0, 0x7f07a64102b8, 0xb)
        /.gvm/gos/go1.3.1/src/pkg/net/fd_unix.go:242 +0x34c
net.(*conn).Read(0xc208024128, 0xc20808b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/net.go:122 +0xe7
net/http.noteEOFReader.Read(0x7f07a6413c28, 0xc208024128, 0xc208072738, 0xc20808b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:1203 +0x72
net/http.(*noteEOFReader).Read(0xc2080aece0, 0xc20808b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        <autogenerated>:124 +0xca
bufio.(*Reader).fill(0xc208004e40)
        /.gvm/gos/go1.3.1/src/pkg/bufio/bufio.go:97 +0x1b3
bufio.(*Reader).Peek(0xc208004e40, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/bufio/bufio.go:132 +0x101
net/http.(*persistConn).readLoop(0xc2080726e0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:782 +0x95
created by net/http.(*Transport).dialConn
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:600 +0x93f

goroutine 30 [select]:
net/http.(*persistConn).writeLoop(0xc2080726e0)
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:885 +0x38f
created by net/http.(*Transport).dialConn
        /.gvm/gos/go1.3.1/src/pkg/net/http/transport.go:601 +0x957

goroutine 49 [select]:
github.com/influxdb/influxdb/cluster.(*WriteBuffer).handleWrites(0xc20842c0e0)
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:75 +0xd3
created by github.com/influxdb/influxdb/cluster.NewWriteBuffer
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:43 +0x286

goroutine 50 [select]:
net.lookupIPDeadline(0xc2080aea80, 0x19, 0xecbd16c33, 0x332d6465, 0x1229980, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/lookup.go:83 +0x414
net.resolveInternetAddr(0xc794d0, 0x3, 0xc2080aea80, 0x1e, 0xecbd16c33, 0x332d6465, 0x1229980, 0x0, 0x0, 0x0, ...)
        /.gvm/gos/go1.3.1/src/pkg/net/ipsock.go:285 +0x49a
net.resolveAddr(0xc72390, 0x4, 0xc794d0, 0x3, 0xc2080aea80, 0x1e, 0xecbd16c33, 0x332d6465, 0x1229980, 0x0, ...)
        /.gvm/gos/go1.3.1/src/pkg/net/dial.go:110 +0x385
net.(*Dialer).Dial(0xc208430000, 0xc794d0, 0x3, 0xc2080aea80, 0x1e, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/dial.go:159 +0xff
net.DialTimeout(0xc794d0, 0x3, 0xc2080aea80, 0x1e, 0x77359400, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/dial.go:151 +0xda
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).reconnect(0xc20803e140, 0x0, 0x0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:226 +0x16b
github.com/influxdb/influxdb/coordinator.func·005()
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:67 +0x2e
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x96

goroutine 51 [sleep]:
time.Sleep(0xdf8475800)
        //.gvm/gos/go1.3.1/src/pkg/runtime/time.goc:39 +0x31
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc20803e140)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:244 +0x37
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0xb1

goroutine 52 [runnable]:
github.com/influxdb/influxdb/cluster.(*ClusterServer).heartbeat(0xc20803a360)
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:149
created by github.com/influxdb/influxdb/cluster.(*ClusterServer).StartHeartbeat
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:91 +0x5a

goroutine 53 [runnable]:
github.com/influxdb/influxdb/cluster.(*WriteBuffer).handleWrites(0xc20842c150)
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:73
created by github.com/influxdb/influxdb/cluster.NewWriteBuffer
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:43 +0x286

goroutine 54 [runnable]:
github.com/influxdb/influxdb/coordinator.func·005()
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:66
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x96

goroutine 55 [runnable]:
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc20803e500)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:242
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0xb1

goroutine 56 [runnable]:
github.com/influxdb/influxdb/cluster.(*ClusterServer).heartbeat(0xc20803a480)
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:149
created by github.com/influxdb/influxdb/cluster.(*ClusterServer).StartHeartbeat
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:91 +0x5a

goroutine 39 [runnable]:
net._C2func_getaddrinfo(0x2161760, 0x0, 0xc2083ee120, 0xc208436008, 0x0, 0x0, 0x0)
        net/_obj/_cgo_defun.c:52 +0x36
net.cgoLookupIPCNAME(0xc2080aea80, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/cgo_unix.go:96 +0x1e1
net.cgoLookupIP(0xc2080aea80, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc208028d90)
        /.gvm/gos/go1.3.1/src/pkg/net/cgo_unix.go:148 +0x69
net.lookupIP(0xc2080aea80, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/lookup_unix.go:64 +0x66
net.func·024(0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/lookup.go:41 +0x54
net.(*singleflight).Do(0x1229080, 0xc2080aea80, 0x19, 0x7f07a627fea0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/singleflight.go:45 +0x232
net.lookupIPMerge(0xc2080aea80, 0x19, 0x0, 0x0, 0x0, 0x0, 0x0)
        /.gvm/gos/go1.3.1/src/pkg/net/lookup.go:42 +0xae
net.func·025()
        /.gvm/gos/go1.3.1/src/pkg/net/lookup.go:80 +0x3e
created by net.lookupIPDeadline
        /.gvm/gos/go1.3.1/src/pkg/net/lookup.go:82 +0x2d8

goroutine 57 [runnable]:
github.com/influxdb/influxdb/cluster.(*WriteBuffer).handleWrites(0xc20842c1c0)
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:73
created by github.com/influxdb/influxdb/cluster.NewWriteBuffer
        /gocodez/src/github.com/influxdb/influxdb/cluster/write_buffer.go:43 +0x286

goroutine 58 [runnable]:
github.com/influxdb/influxdb/coordinator.func·005()
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:66
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:69 +0x96

goroutine 59 [runnable]:
github.com/influxdb/influxdb/coordinator.(*ProtobufClient).peridicallySweepTimedOutRequests(0xc20803e6e0)
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:242
created by github.com/influxdb/influxdb/coordinator.(*ProtobufClient).connect
        /gocodez/src/github.com/influxdb/influxdb/coordinator/protobuf_client.go:70 +0xb1

goroutine 60 [runnable]:
github.com/influxdb/influxdb/cluster.(*ClusterServer).heartbeat(0xc20803a510)
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:149
created by github.com/influxdb/influxdb/cluster.(*ClusterServer).StartHeartbeat
        /gocodez/src/github.com/influxdb/influxdb/cluster/cluster_server.go:91 +0x5a