4538 Deadlock

gistfile1.txt

[monitor] 2015/10/23 14:00:08 failed to store statistics: timeout
^\SIGQUIT: quit
PC=0x6347b m=0

goroutine 0 [idle]:
runtime.mach_semaphore_wait(0xe03, 0x0, 0x33401, 0xc820124180, 0xc989a0, 0xc989a0, 0x55c59, 0xffffffffffffffff, 0x0, 0x7fff5fbff8ec, ...)
	/usr/local/go/src/runtime/sys_darwin_amd64.s:407 +0xb
runtime.semasleep1(0xffffffffffffffff, 0x0)
	/usr/local/go/src/runtime/os1_darwin.go:385 +0xe5
runtime.semasleep.func1()
	/usr/local/go/src/runtime/os1_darwin.go:401 +0x29
runtime.systemstack(0x7fff5fbff8f0)
	/usr/local/go/src/runtime/asm_amd64.s:278 +0xab
runtime.semasleep(0xffffffffffffffff, 0x0)
	/usr/local/go/src/runtime/os1_darwin.go:402 +0x36
runtime.notesleep(0xc99270)
	/usr/local/go/src/runtime/lock_sema.go:169 +0x100
runtime.stopm()
	/usr/local/go/src/runtime/proc1.go:1128 +0x112
runtime.findrunnable(0xc82001ea00, 0x0)
	/usr/local/go/src/runtime/proc1.go:1530 +0x69e
runtime.schedule()
	/usr/local/go/src/runtime/proc1.go:1639 +0x267
runtime.park_m(0xc820124180)
	/usr/local/go/src/runtime/proc1.go:1698 +0x18b
runtime.mcall(0x7fff5fbffa70)
	/usr/local/go/src/runtime/asm_amd64.s:204 +0x5b

goroutine 1 [chan receive, 2 minutes]:
main.(*Main).Run(0xc82b17df10, 0xc82000a280, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/main.go:96 +0xa39
main.main()
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/main.go:46 +0x29b

goroutine 17 [syscall, 2 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 5 [syscall, 2 minutes]:
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
	/usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 8 [IO wait, 2 minutes]:
net.runtime_pollWait(0xf46838, 0x72, 0xc82000a1a0)
	/usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8200171e0, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8200171e0, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc820017180, 0x0, 0xf80000, 0xc82011a000)
	/usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820030128, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc820030128, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:264 +0x3d
github.com/influxdb/influxdb/tcp.(*Mux).Serve(0xc82010a800, 0xf458b8, 0xc820030128, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:48 +0x4b
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:376 +0xafb

goroutine 9 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030130, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*raftLayer).Accept(0xc8201167b0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:2129 +0x51
github.com/hashicorp/raft.(*NetworkTransport).listen(0xc820017260)
	/Users/jason/go/src/github.com/hashicorp/raft/net_transport.go:346 +0x79
created by github.com/hashicorp/raft.NewNetworkTransport
	/Users/jason/go/src/github.com/hashicorp/raft/net_transport.go:138 +0x284

goroutine 10 [select]:
github.com/hashicorp/raft.(*Raft).leaderLoop(0xc82000c540)
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:810 +0xcce
github.com/hashicorp/raft.(*Raft).runLeader(0xc82000c540)
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:782 +0x88e
github.com/hashicorp/raft.(*Raft).run(0xc82000c540)
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:576 +0xbb
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.run)-fm()
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:241 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc82000c540, 0xc820109dc0)
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 11 [select]:
github.com/hashicorp/raft.(*Raft).runFSM(0xc82000c540)
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:490 +0x1022
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runFSM)-fm()
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:242 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc82000c540, 0xc820109dd0)
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 12 [select, 2 minutes]:
github.com/hashicorp/raft.(*Raft).runSnapshots(0xc82000c540)
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:1597 +0x493
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runSnapshots)-fm()
	/Users/jason/go/src/github.com/hashicorp/raft/raft.go:243 +0x20
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc82000c540, 0xc820109de0)
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:152 +0x44
created by github.com/hashicorp/raft.(*raftState).goFunc
	/Users/jason/go/src/github.com/hashicorp/raft/state.go:153 +0x4d

goroutine 13 [select, 2 minutes]:
github.com/influxdb/influxdb/meta.(*localRaft).logLeaderChanges(0xc820015720)
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/state.go:189 +0x4f5
created by github.com/influxdb/influxdb/meta.(*localRaft).open
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/state.go:179 +0x1344

goroutine 14 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030138, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*Store).serveExecListener(0xc82008c780)
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:622 +0xa2
created by github.com/influxdb/influxdb/meta.(*Store).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:239 +0x364

goroutine 15 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030140, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/meta.(*Store).serveRPCListener(0xc82008c780)
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:726 +0xa2
created by github.com/influxdb/influxdb/meta.(*Store).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:242 +0x3b6

goroutine 34 [select, 2 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).monitorErrorChan(0xc820083c20, 0xc82001a720)
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:531 +0x145
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:382 +0xc7f

goroutine 35 [select]:
github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePoints(0xc820080630, 0xc8257d1f80, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:265 +0x651
github.com/influxdb/influxdb/monitor.(*Monitor).storeStatistics(0xc820088620)
	/Users/jason/go/src/github.com/influxdb/influxdb/monitor/service.go:379 +0xdbe
created by github.com/influxdb/influxdb/monitor.(*Monitor).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/monitor/service.go:138 +0x349

goroutine 82 [select, 1 minutes]:
github.com/influxdb/influxdb/tsdb.(*Store).periodicMaintenance(0xc82008c640)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:320 +0x11c
created by github.com/influxdb/influxdb/tsdb.(*Store).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:375 +0x3b7

goroutine 83 [select]:
github.com/influxdb/influxdb/services/hh.(*Service).retryWrites(0xc82008afa0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:136 +0x37e
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:93 +0x237

goroutine 84 [select, 2 minutes]:
github.com/influxdb/influxdb/services/hh.(*Service).expireWrites(0xc82008afa0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:164 +0x2d1
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:94 +0x259

goroutine 85 [select, 2 minutes]:
github.com/influxdb/influxdb/services/hh.(*Service).deleteInactiveQueues(0xc82008afa0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:181 +0x2d1
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/hh/service.go:95 +0x27b

goroutine 86 [chan receive]:
github.com/influxdb/influxdb/services/subscriber.(*Service).writePoints(0xc82001a900)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/subscriber/service.go:204 +0xac
created by github.com/influxdb/influxdb/services/subscriber.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/subscriber/service.go:77 +0x13f

goroutine 87 [select]:
github.com/influxdb/influxdb/meta.(*Store).WaitForDataChanged(0xc82008c780, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/meta/store.go:441 +0x203
github.com/influxdb/influxdb/services/subscriber.(*Service).waitForMetaUpdates(0xc82001a900)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/subscriber/service.go:97 +0x4d
created by github.com/influxdb/influxdb/services/subscriber.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/subscriber/service.go:79 +0x161

goroutine 88 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030148, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/cluster.(*Service).serve(0xc820017110)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/service.go:98 +0xdb
created by github.com/influxdb/influxdb/cluster.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/service.go:75 +0x135

goroutine 89 [select, 2 minutes]:
github.com/influxdb/influxdb/services/precreator.(*Service).runPrecreation(0xc82010a4c0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/precreator/service.go:75 +0x44f
created by github.com/influxdb/influxdb/services/precreator.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/precreator/service.go:53 +0x21f

goroutine 91 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030150, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/services/snapshotter.(*Service).serve(0xc820015270)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/snapshotter/service.go:75 +0xa9
created by github.com/influxdb/influxdb/services/snapshotter.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/snapshotter/service.go:48 +0x130

goroutine 92 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc820030158, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tcp/mux.go:129 +0x6f
github.com/influxdb/influxdb/services/copier.(*Service).serve(0xc82010a540)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/copier/service.go:78 +0xa9
created by github.com/influxdb/influxdb/services/copier.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/copier/service.go:51 +0x130

goroutine 93 [IO wait, 2 minutes]:
net.runtime_pollWait(0xf46778, 0x72, 0xc82000a1a0)
	/usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8203e23e0, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8203e23e0, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc8203e2380, 0x0, 0xf80000, 0xc824e26280)
	/usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820176070, 0x55cc0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc820176070, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:264 +0x3d
net/http.(*Server).Serve(0xc8201fa0c0, 0xf458b8, 0xc820176070, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:1887 +0xb3
net/http.Serve(0xf458b8, 0xc820176070, 0x3862a68, 0xc825a460c0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:1765 +0x85
github.com/influxdb/influxdb/services/admin.(*Service).serve(0xc820015360)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/admin/service.go:107 +0x12f
created by github.com/influxdb/influxdb/services/admin.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/admin/service.go:70 +0x491

goroutine 94 [select]:
github.com/influxdb/influxdb/services/continuous_querier.(*Service).backgroundLoop(0xc820090680)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:184 +0x55f
created by github.com/influxdb/influxdb/services/continuous_querier.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:118 +0x255

goroutine 95 [IO wait]:
net.runtime_pollWait(0xf466b8, 0x72, 0xc82000a1a0)
	/usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8203e2450, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8203e2450, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc8203e23f0, 0x0, 0xf80000, 0xc8546a6de0)
	/usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820176078, 0xc820149dc0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net.(*TCPListener).Accept(0xc820176078, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:264 +0x3d
net/http.(*Server).Serve(0xc8201ee120, 0xf458b8, 0xc820176078, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:1887 +0xb3
net/http.Serve(0xf458b8, 0xc820176078, 0xf7b980, 0xc820090700, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:1765 +0x85
github.com/influxdb/influxdb/services/httpd.(*Service).serve(0xc82001ac60)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/httpd/service.go:133 +0x6e
created by github.com/influxdb/influxdb/services/httpd.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/httpd/service.go:101 +0x6ca

goroutine 96 [select, 2 minutes]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShardGroups(0xc8200154f0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/retention/service.go:69 +0x351
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/retention/service.go:45 +0x1b4

goroutine 97 [select, 2 minutes]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShards(0xc8200154f0)
	/Users/jason/go/src/github.com/influxdb/influxdb/services/retention/service.go:102 +0x7a0
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
	/Users/jason/go/src/github.com/influxdb/influxdb/services/retention/service.go:46 +0x1d9

goroutine 98 [chan receive, 2 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).startServerReporting(0xc820083c20)
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:482 +0x1ee
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open.func1
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:414 +0x13c0

goroutine 99 [select, 2 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Command).monitorServerErrors(0xc820080480)
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:145 +0x2c8
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Command).Run
	/Users/jason/go/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:127 +0xf97

goroutine 100 [select, 2 minutes, locked to thread]:
runtime.gopark(0x99f018, 0xc825b01f28, 0x84a478, 0x6, 0x32018, 0x2)
	/usr/local/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc825b01f28, 0x0, 0x18)
	/usr/local/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc825b01f28)
	/usr/local/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
	/usr/local/go/src/runtime/signal1_unix.go:227 +0x323
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 2888 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc84e51c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0xc83cc91a01, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc84e51c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc84e51c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc84e51c000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc84e51c000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc835088b4f, 0x6, 0xc821851130, 0x7, 0xc8320929c0, 0x1, 0xf, 0x1, 0xc84e51c000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2912 [semacquire]:
sync.runtime_Semacquire(0xc82008c648)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*RWMutex).Lock(0xc82008c640)
	/usr/local/go/src/sync/rwmutex.go:87 +0xa1
github.com/influxdb/influxdb/tsdb.(*Store).CreateShard(0xc82008c640, 0xc8471188df, 0x3, 0xc845ff6db0, 0x7, 0x11, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:78 +0x50
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc8471188df, 0x3, 0xc845ff6db0, 0x7, 0xc8391d29c0, 0x0, 0x11, 0x1, 0xc836f18d20, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:306 +0x1d8
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2932 [semacquire]:
sync.runtime_Semacquire(0xc82008c64c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*RWMutex).RLock(0xc82008c640)
	/usr/local/go/src/sync/rwmutex.go:36 +0x58
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0x1, 0xc82128ae00, 0xb, 0x10, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:382 +0x3c
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0x8578a0, 0x9, 0xc8254bc160, 0x7, 0xc83f916ea0, 0x1, 0x1, 0x1, 0xc82128ae00, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2896 [semacquire]:
sync.runtime_Semacquire(0xc838ec0314)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0310)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).writeToLog(0xc838ec0300, 0x62101, 0xc83554a000, 0x10a77, 0x621a1, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:422 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc85663a000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:215 +0x341
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc85663a000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc85663a000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc85663a000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc847cfceaf, 0x6, 0xc821851130, 0x7, 0xc832093b60, 0x1, 0xf, 0x1, 0xc85663a000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2881 [semacquire]:
sync.runtime_Semacquire(0xc838ec0314)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0310)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).writeToLog(0xc838ec0300, 0x62101, 0xc831b6c000, 0x10ae2, 0x62196, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:422 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc84f87e000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:215 +0x341
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc84f87e000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc84f87e000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc84f87e000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc842871c2f, 0x6, 0xc821851130, 0x7, 0xc83f87c420, 0x1, 0xf, 0x1, 0xc84f87e000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2933 [select]:
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard(0xc820080630, 0xc845969160, 0x8578a0, 0x9, 0xc8254bc160, 0x7, 0x1, 0xc82128b200, 0xb, 0x10, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:341 +0x904
github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePoints.func1(0xc8391d3800, 0xc820080630, 0xc8257d1f80, 0xc845969160, 0x8578a0, 0x9, 0xc8254bc160, 0x7, 0xc82128b200, 0xb, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:250 +0x97
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePoints
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:251 +0x3b1

goroutine 2934 [semacquire]:
sync.runtime_Semacquire(0xc82008c64c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*RWMutex).RLock(0xc82008c640)
	/usr/local/go/src/sync/rwmutex.go:36 +0x58
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0x1, 0xc82128b200, 0xb, 0x10, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:382 +0x3c
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0x8578a0, 0x9, 0xc8254bc160, 0x7, 0xc8391d39e0, 0x1, 0x1, 0x1, 0xc82128b200, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2845 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc8494ea000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc8494ea000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc8494ea000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc8494ea000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc8494ea000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc844c7fa1f, 0x6, 0xc821851130, 0x7, 0xc82a734c60, 0x1, 0xf, 0x1, 0xc8494ea000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2910 [IO wait]:
net.runtime_pollWait(0xf45f38, 0x72, 0xc82000a1a0)
	/usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8380caf40, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8380caf40, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).Read(0xc8380caee0, 0xc825ba6000, 0x1000, 0x1000, 0x0, 0xf40050, 0xc82000a1a0)
	/usr/local/go/src/net/fd_unix.go:232 +0x23a
net.(*conn).Read(0xc8200300e0, 0xc825ba6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:172 +0xe4
net/http.(*liveSwitchReader).Read(0xc8200cfc78, 0xc825ba6000, 0x1000, 0x1000, 0xc825ba7000, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:219 +0xa4
io.(*LimitedReader).Read(0xc8546a6dc0, 0xc825ba6000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:427 +0xbd
bufio.(*Reader).fill(0xc8306ba240)
	/usr/local/go/src/bufio/bufio.go:97 +0x1e9
bufio.(*Reader).Peek(0xc8306ba240, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:132 +0xcc
net/http.(*conn).readRequest(0xc8200cfc30, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/server.go:629 +0x2bb
net/http.(*conn).serve(0xc8200cfc30)
	/usr/local/go/src/net/http/server.go:1319 +0x727
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:1910 +0x3f6

goroutine 2892 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc83a13a000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc83a13a000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc83a13a000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc83a13a000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc83a13a000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc844c7f89f, 0x6, 0xc821851130, 0x7, 0xc832093860, 0x1, 0xf, 0x1, 0xc83a13a000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2865 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc84e62c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc84e62c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc84e62c000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc84e62c000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc84e62c000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc83e4d559f, 0x6, 0xc821851130, 0x7, 0xc83ffba540, 0x1, 0xf, 0x1, 0xc84e62c000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2947 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).flush.func1(0xc838ec0300)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:566 +0x2d
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).flush(0xc838ec0300, 0x1, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:586 +0x13a9
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache.func1(0xc838ec0300)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:275 +0x31
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:278 +0x153

goroutine 2846 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc84c092000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc84c092000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc84c092000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc84c092000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc84c092000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc84f84d86f, 0x6, 0xc821851130, 0x7, 0xc82a734fc0, 0x1, 0xf, 0x1, 0xc84c092000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2917 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc82c424000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x363630393401, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc82c424000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc82c424000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc82c424000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc82c424000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc84f84d5ff, 0x6, 0xc821851130, 0x7, 0xc83a9097a0, 0x1, 0xf, 0x1, 0xc82c424000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

goroutine 2877 [semacquire]:
sync.runtime_Semacquire(0xc838ec033c)
	/usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*Mutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/mutex.go:82 +0x1c4
sync.(*RWMutex).Lock(0xc838ec0338)
	/usr/local/go/src/sync/rwmutex.go:82 +0x30
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).addToCache(0xc838ec0300, 0xc84c496000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0xc837ac3301, 0x0, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:260 +0x55
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Log).WritePoints(0xc838ec0300, 0xc84c496000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:201 +0xa5
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc830bf2b40, 0xc84c496000, 0x1387, 0x1800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tsm1.go:389 +0x8a
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc839d1a6e0, 0xc84c496000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x670
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc82008c640, 0xf, 0xc84c496000, 0x1387, 0x1800, 0x0, 0x0)
	/Users/jason/go/src/github.com/influxdb/influxdb/tsdb/store.go:396 +0x14e
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1(0xc820080630, 0xc842fbe2af, 0x6, 0xc821851130, 0x7, 0xc83f87c0c0, 0x1, 0xf, 0x1, 0xc84c496000, ...)
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:302 +0x10b
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
	/Users/jason/go/src/github.com/influxdb/influxdb/cluster/points_writer.go:334 +0x1c5

rax    0xe
rbx    0xe03
rcx    0x7fff5fbff878
rdx    0x7fff5fbff8f0
rdi    0xe03
rsi    0xc989a0
rbp    0xc99160
rsp    0x7fff5fbff878
r8     0xc99160
r9     0xc82001b260
r10    0x8
r11    0x286
r12    0x7
r13    0x99a597
r14    0x7
r15    0x8
rip    0x6347b
rflags 0x286
cs     0x7
fs     0x0
gs     0x0