tendermint_error.sh

tendermint Issue-#2311 L ✔ ✘-INT
10:33:58 ▶ make test
--> Running go test
ok  	github.com/tendermint/tendermint/abci/client	0.228s
?   	github.com/tendermint/tendermint/abci/cmd/abci-cli	[no test files]
ok  	github.com/tendermint/tendermint/abci/example	2.337s
?   	github.com/tendermint/tendermint/abci/example/code	[no test files]
?   	github.com/tendermint/tendermint/abci/example/counter	[no test files]
ok  	github.com/tendermint/tendermint/abci/example/kvstore	0.039s
?   	github.com/tendermint/tendermint/abci/server	[no test files]
ok  	github.com/tendermint/tendermint/abci/tests	0.005s
?   	github.com/tendermint/tendermint/abci/tests/benchmarks	[no test files]
?   	github.com/tendermint/tendermint/abci/tests/benchmarks/parallel	[no test files]
?   	github.com/tendermint/tendermint/abci/tests/benchmarks/simple	[no test files]
?   	github.com/tendermint/tendermint/abci/tests/server	[no test files]
?   	github.com/tendermint/tendermint/abci/tests/test_app	[no test files]
ok  	github.com/tendermint/tendermint/abci/types	0.023s
?   	github.com/tendermint/tendermint/abci/version	[no test files]
ok  	github.com/tendermint/tendermint/benchmarks	0.010s [no tests to run]
?   	github.com/tendermint/tendermint/benchmarks/proto	[no test files]
?   	github.com/tendermint/tendermint/benchmarks/simu	[no test files]
ok  	github.com/tendermint/tendermint/blockchain	2.062s
?   	github.com/tendermint/tendermint/cmd/priv_val_server	[no test files]
?   	github.com/tendermint/tendermint/cmd/tendermint	[no test files]
ok  	github.com/tendermint/tendermint/cmd/tendermint/commands	0.017s
ok  	github.com/tendermint/tendermint/config	0.007s
ok  	github.com/tendermint/tendermint/consensus	19.643s
ok  	github.com/tendermint/tendermint/consensus/types	0.012s
ok  	github.com/tendermint/tendermint/crypto	0.003s
ok  	github.com/tendermint/tendermint/crypto/armor	0.003s
ok  	github.com/tendermint/tendermint/crypto/ed25519	0.004s
ok  	github.com/tendermint/tendermint/crypto/encoding/amino	0.036s
?   	github.com/tendermint/tendermint/crypto/internal/benchmarking	[no test files]
ok  	github.com/tendermint/tendermint/crypto/merkle	0.008s
ok  	github.com/tendermint/tendermint/crypto/multisig	0.034s
ok  	github.com/tendermint/tendermint/crypto/multisig/bitarray	0.005s
ok  	github.com/tendermint/tendermint/crypto/secp256k1	0.051s
ok  	github.com/tendermint/tendermint/crypto/tmhash	0.004s
ok  	github.com/tendermint/tendermint/crypto/xchacha20poly1305	0.032s
ok  	github.com/tendermint/tendermint/crypto/xsalsa20symmetric	0.411s
?   	github.com/tendermint/tendermint/docs/spec/scripts	[no test files]
ok  	github.com/tendermint/tendermint/evidence	0.247s
ok  	github.com/tendermint/tendermint/libs/autofile	0.607s
?   	github.com/tendermint/tendermint/libs/autofile/cmd	[no test files]
ok  	github.com/tendermint/tendermint/libs/bech32	0.003s
ok  	github.com/tendermint/tendermint/libs/cli	0.012s
ok  	github.com/tendermint/tendermint/libs/cli/flags	0.002s
ok  	github.com/tendermint/tendermint/libs/clist	4.259s
ok  	github.com/tendermint/tendermint/libs/common	4.738s
SIGQUIT: quit
PC=0x45f1d3 m=0 sigcode=0

goroutine 50 [syscall]:
runtime.notetsleepg(0xca0560, 0x773488e1, 0x0)
	/usr/local/go/src/runtime/lock_futex.go:227 +0x37 fp=0xc000048758 sp=0xc000048728 pc=0x40c607
runtime.timerproc(0xca0540)
	/usr/local/go/src/runtime/time.go:288 +0x30e fp=0xc0000487d8 sp=0xc000048758 pc=0x44c8ee
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0000487e0 sp=0xc0000487d8 pc=0x45d381
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 1 [chan receive, 9 minutes]:
testing.(*T).Run(0xc001206100, 0x8eb6d7, 0x14, 0x901310, 0x47c201)
	/usr/local/go/src/testing/testing.go:879 +0x37a
testing.runTests.func1(0xc000138000)
	/usr/local/go/src/testing/testing.go:1119 +0x78
testing.tRunner(0xc000138000, 0xc0000bbe08)
	/usr/local/go/src/testing/testing.go:827 +0xbf
testing.runTests(0xc0000aa560, 0xc96760, 0x1d, 0x1d, 0x40d5df)
	/usr/local/go/src/testing/testing.go:1117 +0x2aa
testing.(*M).Run(0xc0000e6080, 0x0)
	/usr/local/go/src/testing/testing.go:1034 +0x165
main.main()
	_testmain.go:100 +0x13d

goroutine 19 [syscall, 9 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 32 [runnable]:
syscall.Syscall(0x107, 0xffffffffffffff9c, 0xc000ed6040, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.unlinkat(0xffffffffffffff9c, 0xc000ed6000, 0x3d, 0x0, 0x0, 0xc000ed6000)
	/usr/local/go/src/syscall/zsyscall_linux_amd64.go:126 +0x8b
syscall.Unlink(0xc000ed6000, 0x3d, 0x3d, 0xc000ed6000)
	/usr/local/go/src/syscall/syscall_linux.go:168 +0x47
os.Remove(0xc000ed6000, 0x3d, 0x1c, 0x3d)
	/usr/local/go/src/os/file_unix.go:306 +0x39
os.RemoveAll(0xc000ed6000, 0x3d, 0x20, 0x8e546d)
	/usr/local/go/src/os/path.go:68 +0x4d
os.RemoveAll(0xc0000cc260, 0x20, 0x1d, 0x8e560f)
	/usr/local/go/src/os/path.go:109 +0x4f5
github.com/tendermint/tendermint/libs/db.cleanupDBDir(0x0, 0x0, 0xc0000224e0, 0x1d)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/backend_test.go:16 +0xc7
github.com/tendermint/tendermint/libs/db.TestGoLevelDBBackend(0xc001206100)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/backend_test.go:152 +0x174
testing.tRunner(0xc001206100, 0x901310)
	/usr/local/go/src/testing/testing.go:827 +0xbf
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:878 +0x353

goroutine 21 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc0001022a0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x12a
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x177

goroutine 26 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc000170000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0xd3
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:142 +0x40c

goroutine 27 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc000170000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0xe7
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:143 +0x42e

goroutine 28 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc000170000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:834 +0x331
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:149 +0x58c

goroutine 29 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc000170000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:762 +0x12e
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:150 +0x5ae

goroutine 30 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc000102000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x12a
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x177

goroutine 35 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc0000ac000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0xd3
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:142 +0x40c

goroutine 36 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc0000ac000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0xe7
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:143 +0x42e

goroutine 37 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc0000ac000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:834 +0x331
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:149 +0x58c

goroutine 38 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc0000ac000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:762 +0x12e
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:150 +0x5ae

goroutine 83 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc0001701a0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0xe7
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:143 +0x42e

goroutine 85 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc0001701a0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:762 +0x12e
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:150 +0x5ae

goroutine 84 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc0001701a0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:834 +0x331
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:149 +0x58c

goroutine 82 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc0001701a0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0xd3
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:142 +0x40c

goroutine 33 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc0005bc0e0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x12a
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x177

rax    0xfffffffffffffffc
rbx    0x3b99bee1
rcx    0x45f1d3
rdx    0x0
rdi    0xca0560
rsi    0x80
rbp    0xc0000486e0
rsp    0xc000048698
r8     0x0
r9     0x0
r10    0xc0000486d0
r11    0x206
r12    0xc
r13    0xff
r14    0x9540c2
r15    0x0
rip    0x45f1d3
rflags 0x206
cs     0x33
fs     0x0
gs     0x0
*** Test killed with quit: ran too long (10m0s).
FAIL	github.com/tendermint/tendermint/libs/db	600.006s
SIGQUIT: quit
PC=0x45f313 m=0 sigcode=0

goroutine 38 [syscall]:
runtime.notetsleepg(0xf12240, 0x773462ba, 0x0)
	/usr/local/go/src/runtime/lock_futex.go:227 +0x37 fp=0xc0001b9f58 sp=0xc0001b9f28 pc=0x40c747
runtime.timerproc(0xf12220)
	/usr/local/go/src/runtime/time.go:288 +0x30e fp=0xc0001b9fd8 sp=0xc0001b9f58 pc=0x44ca2e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc0001b9fe0 sp=0xc0001b9fd8 pc=0x45d4c1
created by runtime.(*timersBucket).addtimerLocked
	/usr/local/go/src/runtime/time.go:170 +0x114

goroutine 1 [chan receive, 9 minutes]:
testing.(*T).Run(0xc000176100, 0xa40755, 0xc, 0xa61ae8, 0x47cc26)
	/usr/local/go/src/testing/testing.go:879 +0x37a
testing.runTests.func1(0xc000176000)
	/usr/local/go/src/testing/testing.go:1119 +0x78
testing.tRunner(0xc000176000, 0xc0000bbe08)
	/usr/local/go/src/testing/testing.go:827 +0xbf
testing.runTests(0xc0001686c0, 0xefacd0, 0x1, 0x1, 0x40d71f)
	/usr/local/go/src/testing/testing.go:1117 +0x2aa
testing.(*M).Run(0xc0000e6080, 0x0)
	/usr/local/go/src/testing/testing.go:1034 +0x165
main.main()
	_testmain.go:44 +0x13d

goroutine 19 [syscall, 9 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 20 [runnable]:
syscall.Syscall(0x107, 0xffffffffffffff9c, 0xc0001e8000, 0x200, 0xffffffffffffffff, 0x0, 0x27)
	/usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5
syscall.unlinkat(0xffffffffffffff9c, 0xc0001e8640, 0x11, 0x200, 0xad1700, 0xc000296060)
	/usr/local/go/src/syscall/zsyscall_linux_amd64.go:126 +0x8b
syscall.Rmdir(0xc0001e8640, 0x11, 0xad1700, 0xc000296060)
	/usr/local/go/src/syscall/syscall_linux.go:158 +0x47
os.Remove(0xc0001e8640, 0x11, 0x0, 0x0)
	/usr/local/go/src/os/file_unix.go:310 +0x6e
os.RemoveAll(0xc0001e8640, 0x11, 0xe, 0xa3a7b0)
	/usr/local/go/src/os/path.go:131 +0x2da
github.com/tendermint/tendermint/libs/db/remotedb_test.TestRemoteDB.func2(0xa41c6a, 0xe)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/remotedb/remotedb_test.go:33 +0x6f
github.com/tendermint/tendermint/libs/db/remotedb_test.TestRemoteDB(0xc000176100)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/remotedb/remotedb_test.go:123 +0x231c
testing.tRunner(0xc000176100, 0xa61ae8)
	/usr/local/go/src/testing/testing.go:827 +0xbf
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:878 +0x353

goroutine 21 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7f141d95bf00, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc0000e6118, 0x72, 0xc00006c000, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc0000e6118, 0xffffffffffffff00, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc0000e6100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(*netFD).accept(0xc0000e6100, 0xc0001ac000, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00009e070, 0xc000059e78, 0xc000059e80, 0x18)
	/usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
net.(*TCPListener).Accept(0xc00009e070, 0xa62b88, 0xc000104700, 0xad8e00, 0xc0001ac000)
	/usr/local/go/src/net/tcpsock.go:260 +0x47
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*Server).Serve(0xc000104700, 0xad5d20, 0xc00009e070, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/server.go:544 +0x210
github.com/tendermint/tendermint/libs/db/remotedb_test.TestRemoteDB.func1(0xc000104700, 0xad5d20, 0xc00009e070, 0xc000176100)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/remotedb/remotedb_test.go:23 +0x43
created by github.com/tendermint/tendermint/libs/db/remotedb_test.TestRemoteDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/libs/db/remotedb/remotedb_test.go:22 +0x20c

goroutine 22 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc00016aae0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:109 +0x142
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*ccResolverWrapper).start
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:95 +0x3f

goroutine 23 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000a2dc0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x10f
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc.newCCBalancerWrapper
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x13d

goroutine 24 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc0000f9600)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/clientconn.go:1384 +0x1c1
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc0000f9600)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/clientconn.go:960 +0x1b2
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*addrConn).connect
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/clientconn.go:951 +0xe1

goroutine 5 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7f141d95be30, 0x72, 0xc0001d8988)
	/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc0000e6418, 0x72, 0xffffffffffffff00, 0xad1700, 0xec27c0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc0000e6418, 0xc0001d0400, 0x400, 0x400)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc0000e6400, 0xc0001d0400, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x179
net.(*netFD).Read(0xc0000e6400, 0xc0001d0400, 0x400, 0x400, 0x40a60b, 0xc000010000, 0x999f20)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e010, 0xc0001d0400, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:177 +0x68
crypto/tls.(*block).readFromUntil(0xc0001bc090, 0xad0660, 0xc00000e010, 0x5, 0xc00000e010, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:492 +0x89
crypto/tls.(*Conn).readRecord(0xc00004b500, 0xa63717, 0xc00004b620, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:593 +0xdd
crypto/tls.(*Conn).Read(0xc00004b500, 0xc000268000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1145 +0xf1
bufio.(*Reader).Read(0xc00006e300, 0xc00023a118, 0x9, 0x9, 0x9, 0x0, 0x0)
	/usr/local/go/src/bufio/bufio.go:216 +0x22f
io.ReadAtLeast(0xacfcc0, 0xc00006e300, 0xc00023a118, 0x9, 0x9, 0x9, 0xc0000d0390, 0x4032fe, 0xc0001d8e9b)
	/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(0xacfcc0, 0xc00006e300, 0xc00023a118, 0x9, 0x9, 0x32e638f4, 0x32e638f4002a86ec, 0x5b920975)
	/usr/local/go/src/io/io.go:329 +0x58
github.com/tendermint/tendermint/vendor/golang.org/x/net/http2.readFrameHeader(0xc00023a118, 0x9, 0x9, 0xacfcc0, 0xc00006e300, 0x0, 0xbedca03d00000000, 0x4cd8fe8, 0xf0df40)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/tendermint/tendermint/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023a0e0, 0xc0002a86e0, 0xc0002a86e0, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/golang.org/x/net/http2/frame.go:492 +0xa3
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc000280000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_client.go:1135 +0x12b
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_client.go:270 +0xad7

goroutine 6 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc00006c2c0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/controlbuf.go:300 +0x103
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc000246240, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/controlbuf.go:387 +0x1ac
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc000280000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_client.go:303 +0x7b
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Client
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_client.go:301 +0xc3a

goroutine 35 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*controlBuffer).get(0xc0001de140, 0x1, 0x0, 0x0, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/controlbuf.go:300 +0x103
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*loopyWriter).run(0xc000246360, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/controlbuf.go:387 +0x1ac
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc000264000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_server.go:277 +0xcb
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_server.go:274 +0xd3b

goroutine 36 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc000264000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_server.go:875 +0x1ea
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.newHTTP2Server
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_server.go:283 +0xd60

goroutine 37 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7f141d95bd60, 0x72, 0xc0001d4908)
	/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc0001a8018, 0x72, 0xffffffffffffff00, 0xad1700, 0xec27c0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc0001a8018, 0xc0001e2800, 0x400, 0x400)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc0001a8000, 0xc0001e2800, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x179
net.(*netFD).Read(0xc0001a8000, 0xc0001e2800, 0x400, 0x400, 0x40a60b, 0xc000010000, 0x999f20)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0001ac000, 0xc0001e2800, 0x400, 0x400, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:177 +0x68
crypto/tls.(*block).readFromUntil(0xc0001e00f0, 0xad0660, 0xc0001ac000, 0x5, 0xc0001ac000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:492 +0x89
crypto/tls.(*Conn).readRecord(0xc0001be380, 0xa63717, 0xc0001be4a0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:593 +0xdd
crypto/tls.(*Conn).Read(0xc0001be380, 0xc00024a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1145 +0xf1
bufio.(*Reader).Read(0xc0002461e0, 0xc000262038, 0x9, 0x9, 0x18, 0x9ba620, 0x0)
	/usr/local/go/src/bufio/bufio.go:216 +0x22f
io.ReadAtLeast(0xacfcc0, 0xc0002461e0, 0xc000262038, 0x9, 0x9, 0x9, 0x40d71f, 0x4032fe, 0xc0001d4d80)
	/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(0xacfcc0, 0xc0002461e0, 0xc000262038, 0x9, 0x9, 0x6, 0xc00003af88, 0xc00003af00)
	/usr/local/go/src/io/io.go:329 +0x58
github.com/tendermint/tendermint/vendor/golang.org/x/net/http2.readFrameHeader(0xc000262038, 0x9, 0x9, 0xacfcc0, 0xc0002461e0, 0x0, 0xc000000000, 0xc0001d4e18, 0xc0001d4e90)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/tendermint/tendermint/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc000262000, 0xc0002a86c0, 0xc0002a86c0, 0x0, 0x0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/golang.org/x/net/http2/frame.go:492 +0xa3
github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc000264000, 0xc0001aa090, 0xa62be8)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/transport/http2_server.go:434 +0x7c
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc000104700, 0xad96a0, 0xc000264000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/server.go:676 +0xdd
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2()
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/server.go:631 +0x3c
github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc000288040, 0xc000104700, 0x7f141d897000, 0xc000264000)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/server.go:639 +0x27
created by github.com/tendermint/tendermint/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/google.golang.org/grpc/server.go:638 +0x5f8

goroutine 28 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc000102540)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x12a
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x177

goroutine 8 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc0000ad1e0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0xd3
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:142 +0x40c

goroutine 9 [select]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc0000ad1e0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:101 +0xe7
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:143 +0x42e

goroutine 10 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc0000ad1e0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:834 +0x331
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:149 +0x58c

goroutine 11 [select, 9 minutes]:
github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc0000ad1e0)
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:762 +0x12e
created by github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb.openDB
	/mnt/lighthouse/gowork/src/github.com/tendermint/tendermint/vendor/github.com/syndtr/goleveldb/leveldb/db.go:150 +0x5ae

rax    0xfffffffffffffffc
rbx    0x3b9998ba
rcx    0x45f313
rdx    0x0
rdi    0xf12240
rsi    0x80
rbp    0xc0001b9ee0
rsp    0xc0001b9e98
r8     0x0
r9     0x0
r10    0xc0001b9ed0
r11    0x202
r12    0x4a7311b7394
r13    0x4
r14    0xc000049700
r15    0x0
rip    0x45f313
rflags 0x202
cs     0x33
fs     0x0
gs     0x0
*** Test killed with quit: ran too long (10m0s).
FAIL	github.com/tendermint/tendermint/libs/db/remotedb	600.006s
ok  	github.com/tendermint/tendermint/libs/db/remotedb/grpcdb	0.005s [no tests to run]
?   	github.com/tendermint/tendermint/libs/db/remotedb/proto	[no test files]
ok  	github.com/tendermint/tendermint/libs/events	0.013s
ok  	github.com/tendermint/tendermint/libs/flowrate	0.942s
ok  	github.com/tendermint/tendermint/libs/log	0.013s
ok  	github.com/tendermint/tendermint/libs/pubsub	0.019s
ok  	github.com/tendermint/tendermint/libs/pubsub/query	0.012s
?   	github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test	[no test files]
?   	github.com/tendermint/tendermint/libs/test	[no test files]
?   	github.com/tendermint/tendermint/libs/version	[no test files]
ok  	github.com/tendermint/tendermint/lite	0.430s
ok  	github.com/tendermint/tendermint/lite/client	5.566s
?   	github.com/tendermint/tendermint/lite/errors	[no test files]
ok  	github.com/tendermint/tendermint/lite/proxy	0.101s
ok  	github.com/tendermint/tendermint/mempool	3.185s
ok  	github.com/tendermint/tendermint/node	0.093s
ok  	github.com/tendermint/tendermint/p2p	2.783s
ok  	github.com/tendermint/tendermint/p2p/conn	1.056s
?   	github.com/tendermint/tendermint/p2p/dummy	[no test files]
ok  	github.com/tendermint/tendermint/p2p/pex	10.041s
ok  	github.com/tendermint/tendermint/p2p/trust	0.054s
?   	github.com/tendermint/tendermint/p2p/upnp	[no test files]
ok  	github.com/tendermint/tendermint/privval	0.179s
ok  	github.com/tendermint/tendermint/proxy	0.009s
ok  	github.com/tendermint/tendermint/rpc/client	5.611s
ok  	github.com/tendermint/tendermint/rpc/client/mock	0.013s
ok  	github.com/tendermint/tendermint/rpc/core	0.018s
ok  	github.com/tendermint/tendermint/rpc/core/types	0.009s
ok  	github.com/tendermint/tendermint/rpc/grpc	0.154s
ok  	github.com/tendermint/tendermint/rpc/lib	8.028s
ok  	github.com/tendermint/tendermint/rpc/lib/client	9.381s
ok  	github.com/tendermint/tendermint/rpc/lib/server	0.041s
?   	github.com/tendermint/tendermint/rpc/lib/test	[no test files]
ok  	github.com/tendermint/tendermint/rpc/lib/types	0.003s
?   	github.com/tendermint/tendermint/rpc/test	[no test files]
?   	github.com/tendermint/tendermint/scripts	[no test files]
?   	github.com/tendermint/tendermint/scripts/json2wal	[no test files]
?   	github.com/tendermint/tendermint/scripts/wal2json	[no test files]
ok  	github.com/tendermint/tendermint/state	0.028s
?   	github.com/tendermint/tendermint/state/txindex	[no test files]
ok  	github.com/tendermint/tendermint/state/txindex/kv	0.008s
?   	github.com/tendermint/tendermint/state/txindex/null	[no test files]
?   	github.com/tendermint/tendermint/test/app	[no test files]
ok  	github.com/tendermint/tendermint/tools/tm-bench	0.514s
?   	github.com/tendermint/tendermint/tools/tm-monitor	[no test files]
?   	github.com/tendermint/tendermint/tools/tm-monitor/eventmeter	[no test files]
?   	github.com/tendermint/tendermint/tools/tm-monitor/mock	[no test files]
ok  	github.com/tendermint/tendermint/tools/tm-monitor/monitor	2.215s
ok  	github.com/tendermint/tendermint/types	0.981s
?   	github.com/tendermint/tendermint/types/proto3	[no test files]
ok  	github.com/tendermint/tendermint/types/time	0.003s
?   	github.com/tendermint/tendermint/version	[no test files]
Makefile:206: recipe for target 'test' failed
make: *** [test] Error 1