Skip to content

Instantly share code, notes, and snippets.

@tamird

tamird/morelogs.diff

Created November 10, 2016 19:14
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tamird/f25af8a4746b2109d1e6fe46f0eb9989 to your computer and use it in GitHub Desktop.
Save tamird/f25af8a4746b2109d1e6fe46f0eb9989 to your computer and use it in GitHub Desktop.
Download ZIP
TestRaftRemoveRace
Raw
morelogs.diff
diff --git a/pkg/storage/client_raft_test.go b/pkg/storage/client_raft_test.go
index 2fcf128..45ebfd9 100644
--- a/pkg/storage/client_raft_test.go
+++ b/pkg/storage/client_raft_test.go
@@ -1999,7 +1999,7 @@ func TestRaftRemoveRace(t *testing.T) {
rangeID := roachpb.RangeID(1)
mtc.replicateRange(rangeID, 1, 2)
- for i := 0; i < 10; i++ {
+ for i := 0; i < 100; i++ {
mtc.unreplicateRange(rangeID, 2)
mtc.replicateRange(rangeID, 2)
}
diff --git a/pkg/storage/store.go b/pkg/storage/store.go
index 24af293..5f3b684 100644
--- a/pkg/storage/store.go
+++ b/pkg/storage/store.go
@@ -1904,6 +1904,8 @@ func (s *Store) addPlaceholderLocked(placeholder *ReplicaPlaceholder) error {
if exRng, ok := s.mu.replicaPlaceholders[rangeID]; ok {
return errors.Errorf("%s has ID collision with existing KeyRange %s", placeholder, exRng)
}
+ ctx := s.AnnotateCtx(context.TODO())
+ log.Infof(ctx, "ADD %s", rangeID)
s.mu.replicaPlaceholders[rangeID] = placeholder
return nil
}
@@ -2025,13 +2027,14 @@ func (s *Store) removeReplicaImpl(
delete(s.mu.replicas, rep.RangeID)
delete(s.mu.replicaQueues, rep.RangeID)
delete(s.mu.uninitReplicas, rep.RangeID)
+ ctx := rep.AnnotateCtx(context.TODO())
if placeholder := s.mu.replicasByKey.Delete(rep); placeholder != rep {
- ctx := rep.AnnotateCtx(context.TODO())
// We already checked that our replica was present in replicasByKey
// above. Nothing should have been able to change that.
log.Fatalf(ctx, "unexpectedly overlapped by %v", placeholder)
}
delete(s.mu.replicaPlaceholders, rep.RangeID)
+ log.Infof(ctx, "DEL %s", rep.RangeID)
s.scanner.RemoveReplica(rep)
s.consistencyScanner.RemoveReplica(rep)
return nil
@@ -2781,6 +2784,7 @@ func (s *Store) processRaftRequest(
defer func() {
if removePlaceholder {
if s.removePlaceholder(req.RangeID) {
+ log.Infof(ctx, "DEL %s", req.RangeID)
atomic.AddInt32(&s.counts.removedPlaceholders, 1)
}
}
@@ -2806,6 +2810,7 @@ func (s *Store) processRaftRequest(
if !s.removePlaceholderLocked(req.RangeID) {
log.Fatalf(ctx, "could not remove placeholder after preemptive snapshot")
}
+ log.Infof(ctx, "DEL %s", req.RangeID)
if pErr == nil {
atomic.AddInt32(&s.counts.filledPlaceholders, 1)
} else {
@@ -3207,6 +3212,8 @@ func (s *Store) processReady(rangeID roachpb.RangeID) {
// placeholder itself, that isn't guaranteed and so this invocation
// here is crucial (i.e. don't remove it).
if s.removePlaceholder(r.RangeID) {
+ ctx := s.AnnotateCtx(context.TODO())
+ log.Infof(ctx, "DEL %s", r.RangeID)
atomic.AddInt32(&s.counts.droppedPlaceholders, 1)
}
}
Raw
stress.log
make stress STRESSFLAGS='-maxfails 1 -stderr -p 16' PKG=./pkg/storage TESTS=TestRaftRemoveRace TESTFLAGS=--vmodule=raft=4,raft_log_queue=1
0 runs so far, 0 failures, over 5s
0 runs so far, 0 failures, over 10s
3 runs so far, 0 failures, over 15s
16 runs so far, 0 failures, over 20s
16 runs so far, 0 failures, over 25s
21 runs so far, 0 failures, over 30s
32 runs so far, 0 failures, over 35s
32 runs so far, 0 failures, over 40s
37 runs so far, 0 failures, over 45s
48 runs so far, 0 failures, over 50s
I161110 14:11:13.109086 1 rand.go:76 Random seed: 244734399514133537
I161110 14:11:13.111243 7 storage/engine/rocksdb.go:340 opening in memory rocksdb instance
I161110 14:11:13.113331 7 gossip/gossip.go:237 [n?] initial resolvers: []
W161110 14:11:13.113374 7 gossip/gossip.go:1055 [n?] no resolvers found; use --join to specify a connected node
I161110 14:11:13.113432 7 base/node_id.go:62 NodeID set to 1
I161110 14:11:13.116796 7 storage/store.go:1183 [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I161110 14:11:13.116929 7 gossip/gossip.go:280 [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:63217" > attrs:<> locality:<>
I161110 14:11:13.117373 24 raft/raft.go:568 [s1,r1/1:/M{in-ax}] 1 became follower at term 5
I161110 14:11:13.117451 24 raft/raft.go:316 [s1,r1/1:/M{in-ax}] newRaft 1 [peers: [1], term: 5, commit: 10, applied: 10, lastindex: 10, lastterm: 5]
I161110 14:11:13.117477 24 raft/raft.go:750 [s1,r1/1:/M{in-ax}] 1 is starting a new election at term 5
I161110 14:11:13.117494 24 raft/raft.go:581 [s1,r1/1:/M{in-ax}] 1 became candidate at term 6
I161110 14:11:13.117512 24 raft/raft.go:665 [s1,r1/1:/M{in-ax}] 1 received MsgVoteResp from 1 at term 6
I161110 14:11:13.117531 24 raft/raft.go:622 [s1,r1/1:/M{in-ax}] 1 became leader at term 6
I161110 14:11:13.118265 52 storage/replica_proposal.go:328 [s1,r1/1:/M{in-ax}] new range lease replica {1 1 1} 1970-01-01 00:00:00 +0000 UTC 900.000124ms following replica {0 0 0} 1970-01-01 00:00:00 +0000 UTC 0s [physicalTime=1970-01-01 00:00:00.000000123 +0000 UTC]
I161110 14:11:13.118871 7 storage/engine/rocksdb.go:340 opening in memory rocksdb instance
I161110 14:11:13.119997 7 gossip/gossip.go:237 [n?] initial resolvers: [127.0.0.1:63217]
W161110 14:11:13.120025 7 gossip/gossip.go:1057 [n?] no incoming or outgoing connections
I161110 14:11:13.120059 7 base/node_id.go:62 NodeID set to 2
I161110 14:11:13.121226 7 storage/store.go:1183 [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I161110 14:11:13.121273 7 gossip/gossip.go:280 [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:63219" > attrs:<> locality:<>
I161110 14:11:13.121737 137 gossip/client.go:125 [n2] started gossip client to 127.0.0.1:63217
I161110 14:11:13.121921 7 storage/engine/rocksdb.go:340 opening in memory rocksdb instance
I161110 14:11:13.123090 7 gossip/gossip.go:237 [n?] initial resolvers: [127.0.0.1:63217]
W161110 14:11:13.123114 7 gossip/gossip.go:1057 [n?] no incoming or outgoing connections
I161110 14:11:13.123149 7 base/node_id.go:62 NodeID set to 3
I161110 14:11:13.126167 7 storage/store.go:1183 [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I161110 14:11:13.126210 7 gossip/gossip.go:280 [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:63221" > attrs:<> locality:<>
I161110 14:11:13.126522 159 gossip/client.go:125 [n3] started gossip client to 127.0.0.1:63217
I161110 14:11:13.127533 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot ce5c48ae at index 14 in 41.805µs.
I161110 14:11:13.128053 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 33, log entries: 4
I161110 14:11:13.128226 205 storage/store.go:1908 [s2] ADD 1
I161110 14:11:13.128264 205 raft/raft.go:568 [s2,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.128365 205 raft/raft.go:316 [s2,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.128378 205 raft/raft.go:568 [s2,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.128406 205 raft/raft.go:710 [s2,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.128417 205 raft/raft.go:568 [s2,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.128473 205 raft/raft.go:1125 [s2,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 14, term: 6]
I161110 14:11:13.128498 205 raft/log.go:288 [s2,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 14, term: 6]
I161110 14:11:13.128524 205 raft/raft.go:1135 [s2,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 15, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.128540 205 raft/raft.go:1102 [s2,r1/?:{-}] ffffffffffffffff [commit: 14] restored snapshot [index: 14, term: 6]
I161110 14:11:13.128564 205 storage/replica_raftstorage.go:589 [s2,r1/?:{-}] applying preemptive snapshot at index 14 (id=ce5c48ae, encoded size=16, 1 rocksdb batches, 4 log entries)
I161110 14:11:13.128929 205 storage/replica_raftstorage.go:592 [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 0.000s
I161110 14:11:13.128952 205 storage/store.go:2813 [s2,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.129467 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2
I161110 14:11:13.130410 216 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.131387 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 7d4ed99e at index 17 in 34.494µs.
I161110 14:11:13.131972 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 37, log entries: 7
I161110 14:11:13.132123 232 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.132162 232 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.132245 232 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.132257 232 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.132284 232 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.132294 232 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.132327 232 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 17, term: 6]
I161110 14:11:13.132340 232 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 17, term: 6]
I161110 14:11:13.132356 232 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 18, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.132370 232 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 18, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.132382 232 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 17] restored snapshot [index: 17, term: 6]
I161110 14:11:13.132396 232 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 17 (id=7d4ed99e, encoded size=16, 1 rocksdb batches, 7 log entries)
I161110 14:11:13.132840 232 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.000s
I161110 14:11:13.132856 232 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.133405 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3
I161110 14:11:13.134245 119 raft/raft.go:568 [s2,r1/?:/M{in-ax}] 2 became follower at term 6
I161110 14:11:13.134337 119 raft/raft.go:316 [s2,r1/?:/M{in-ax}] newRaft 2 [peers: [1], term: 6, commit: 14, applied: 14, lastindex: 14, lastterm: 6]
I161110 14:11:13.134362 119 raft/raft.go:1088 [s2,r1/?:/M{in-ax}] 2 [logterm: 0, index: 17] rejected msgApp [logterm: 6, index: 17] from 1
I161110 14:11:13.134559 177 storage/raft_transport.go:432 raft transport stream to node 1 established
I161110 14:11:13.134716 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 14) from 2 for index 17
I161110 14:11:13.134742 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2 to [next = 15, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.136068 248 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}]
I161110 14:11:13.137548 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:4
I161110 14:11:13.138701 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 3 became follower at term 6
I161110 14:11:13.138794 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 3 [peers: [1,2], term: 6, commit: 17, applied: 17, lastindex: 17, lastterm: 6]
I161110 14:11:13.138811 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 3 [logterm: 0, index: 19] rejected msgApp [logterm: 6, index: 19] from 1
I161110 14:11:13.139007 208 storage/raft_transport.go:432 raft transport stream to node 1 established
I161110 14:11:13.139461 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 17) from 3 for index 19
I161110 14:11:13.139484 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 3 to [next = 18, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.139969 276 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:3}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.146185 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot cdcbbb82 at index 21 in 39.58µs.
I161110 14:11:13.146667 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 43, log entries: 11
I161110 14:11:13.146842 274 storage/store.go:2984 [s3,r1/3:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.146872 274 storage/store.go:2984 [s3,r1/3:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.147291 283 raft/raft.go:568 [s3,r1/3:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:13.147320 283 raft/raft.go:316 [s3,r1/3:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 21, applied: 21, lastindex: 21, lastterm: 6]
I161110 14:11:13.147372 283 raft/raft.go:1106 [s3,r1/3:/M{in-ax}] ffffffffffffffff [commit: 21] ignored snapshot [index: 21, term: 6]
I161110 14:11:13.147595 201 storage/store.go:2037 [s3,r1/3:/M{in-ax}] DEL 1
I161110 14:11:13.147821 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:4
I161110 14:11:13.149095 241 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:4}]
I161110 14:11:13.304258 77 raft/raft.go:568 [s3,r1/4:{-}] 4 became follower at term 0
I161110 14:11:13.304344 77 raft/raft.go:316 [s3,r1/4:{-}] newRaft 4 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.304359 77 raft/raft.go:568 [s3,r1/4:{-}] 4 became follower at term 1
I161110 14:11:13.304380 77 raft/raft.go:710 [s3,r1/4:{-}] 4 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:13.304391 77 raft/raft.go:568 [s3,r1/4:{-}] 4 became follower at term 6
I161110 14:11:13.304406 77 raft/raft.go:1088 [s3,r1/4:{-}] 4 [logterm: 0, index: 23] rejected msgApp [logterm: 6, index: 23] from 1
I161110 14:11:13.304565 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 4 for index 23
I161110 14:11:13.304587 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 4 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.304669 52 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 8f829811 at index 23 in 46.589µs.
I161110 14:11:13.304714 52 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 23] sent snapshot[index: 23, term: 6] to 4 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.304737 52 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 4 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 23]
I161110 14:11:13.306786 256 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.306878 256 raft/raft.go:1125 [s3,r1/4:{-}] 4 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 23, term: 6]
I161110 14:11:13.306894 256 raft/log.go:288 [s3,r1/4:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 23, term: 6]
I161110 14:11:13.306911 256 raft/raft.go:1135 [s3,r1/4:{-}] 4 restored progress of 1 [next = 24, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.306925 256 raft/raft.go:1135 [s3,r1/4:{-}] 4 restored progress of 2 [next = 24, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.306938 256 raft/raft.go:1135 [s3,r1/4:{-}] 4 restored progress of 4 [next = 24, match = 23, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.306948 256 raft/raft.go:1102 [s3,r1/4:{-}] 4 [commit: 23] restored snapshot [index: 23, term: 6]
I161110 14:11:13.306969 256 storage/replica_raftstorage.go:589 [s3,r1/4:{-}] applying Raft snapshot at index 23 (id=8f829811, encoded size=16, 1 rocksdb batches, 13 log entries)
I161110 14:11:13.307362 306 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 46, log entries: 13
I161110 14:11:13.307627 256 storage/replica_raftstorage.go:592 [s3,r1/4:/M{in-ax}] applied Raft snapshot in 0.001s
I161110 14:11:13.307915 53 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 4 [next = 24, match = 23, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 23]
I161110 14:11:13.313159 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:4 > next_replica_id:5
I161110 14:11:13.314504 327 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.316179 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 2ed019f2 at index 28 in 38.204µs.
I161110 14:11:13.316711 201 storage/store.go:2037 [s3,r1/4:/M{in-ax}] DEL 1
I161110 14:11:13.316874 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 52, log entries: 18
I161110 14:11:13.317079 334 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.317130 334 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.317224 334 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.317238 334 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.317259 334 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.317271 334 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.317335 334 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 28, term: 6]
I161110 14:11:13.317350 334 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 28, term: 6]
I161110 14:11:13.317366 334 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 29, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.317379 334 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 29, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.317390 334 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 28] restored snapshot [index: 28, term: 6]
I161110 14:11:13.317404 334 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 28 (id=2ed019f2, encoded size=16, 1 rocksdb batches, 18 log entries)
I161110 14:11:13.318061 334 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161110 14:11:13.318078 334 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.318549 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:5
I161110 14:11:13.318769 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 4 became follower at term 6
I161110 14:11:13.318814 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 4 [peers: [1,2], term: 6, commit: 28, applied: 28, lastindex: 28, lastterm: 6]
I161110 14:11:13.319796 310 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:5}]
I161110 14:11:13.321435 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:5 > next_replica_id:6
I161110 14:11:13.322553 76 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 5 became follower at term 6
I161110 14:11:13.322606 76 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 5 [peers: [1,2], term: 6, commit: 28, applied: 28, lastindex: 28, lastterm: 6]
I161110 14:11:13.322630 76 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 5 [logterm: 0, index: 30] rejected msgApp [logterm: 6, index: 30] from 1
I161110 14:11:13.322804 57 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 28) from 5 for index 30
I161110 14:11:13.322828 57 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 5 to [next = 29, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.327399 274 storage/store.go:2984 [s3,r1/5:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.327748 335 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.329140 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 1e5043e8 at index 32 in 37.553µs.
I161110 14:11:13.330999 201 storage/store.go:2037 [s3,r1/5:/M{in-ax}] DEL 1
I161110 14:11:13.332454 370 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.332524 370 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.332653 370 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.332669 370 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.332695 370 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.332707 370 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.332929 370 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 32, term: 6]
I161110 14:11:13.332954 370 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 32, term: 6]
I161110 14:11:13.332985 370 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 33, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.333001 370 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 33, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.333012 370 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 32] restored snapshot [index: 32, term: 6]
I161110 14:11:13.333036 370 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 32 (id=1e5043e8, encoded size=16, 1 rocksdb batches, 22 log entries)
I161110 14:11:13.333802 370 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161110 14:11:13.333820 370 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.333977 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 58, log entries: 22
I161110 14:11:13.372075 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:6
I161110 14:11:13.377237 365 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:6}]
I161110 14:11:13.538344 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:6 > next_replica_id:7
I161110 14:11:13.545008 79 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 6 became follower at term 6
I161110 14:11:13.545191 79 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 6 [peers: [1,2], term: 6, commit: 32, applied: 32, lastindex: 32, lastterm: 6]
I161110 14:11:13.545215 79 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 6 [logterm: 0, index: 34] rejected msgApp [logterm: 6, index: 34] from 1
I161110 14:11:13.547052 349 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.548000 79 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 6 [logterm: 0, index: 34] rejected msgApp [logterm: 6, index: 34] from 1
I161110 14:11:13.550394 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.550619 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot fbd12421 at index 37 in 33.977µs.
I161110 14:11:13.551352 201 storage/store.go:2037 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.552000 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 65, log entries: 27
I161110 14:11:13.552258 412 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.552312 412 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.552416 412 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.552429 412 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.552448 412 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.552459 412 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.552948 412 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 37, term: 6]
I161110 14:11:13.552966 412 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 37, term: 6]
I161110 14:11:13.552991 412 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 38, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.553006 412 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 38, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.553018 412 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 37] restored snapshot [index: 37, term: 6]
I161110 14:11:13.553034 412 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 37 (id=fbd12421, encoded size=16, 1 rocksdb batches, 27 log entries)
I161110 14:11:13.553921 412 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161110 14:11:13.553940 412 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.554603 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:7
I161110 14:11:13.558928 394 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:7}]
I161110 14:11:13.582145 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:7 > next_replica_id:8
I161110 14:11:13.583686 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 7 became follower at term 6
I161110 14:11:13.583753 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 7 [peers: [1,2], term: 6, commit: 37, applied: 37, lastindex: 37, lastterm: 6]
I161110 14:11:13.583784 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 7 [logterm: 0, index: 41] rejected msgApp [logterm: 6, index: 41] from 1
I161110 14:11:13.588992 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 37) from 7 for index 41
I161110 14:11:13.589089 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 7 to [next = 38, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.589530 380 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.591309 274 storage/store.go:2984 [s3,r1/7:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.591843 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot b563c2f7 at index 43 in 38.354µs.
I161110 14:11:13.592355 201 storage/store.go:2037 [s3,r1/7:/M{in-ax}] DEL 1
I161110 14:11:13.592460 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 73, log entries: 33
W161110 14:11:13.592556 266 storage/store.go:2988 [s1] got error from range 1, replica {3 3 6}: raft group deleted
I161110 14:11:13.592634 437 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.592678 437 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.592797 437 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.592810 437 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.592829 437 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.592839 437 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.592937 437 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 43, term: 6]
I161110 14:11:13.592951 437 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 43, term: 6]
I161110 14:11:13.592966 437 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 44, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.592984 437 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 44, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.592995 437 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 43] restored snapshot [index: 43, term: 6]
I161110 14:11:13.593007 437 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 43 (id=b563c2f7, encoded size=16, 1 rocksdb batches, 33 log entries)
I161110 14:11:13.594035 437 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161110 14:11:13.594052 437 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.594535 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:8
I161110 14:11:13.595661 400 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:8}]
I161110 14:11:13.597407 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:8 > next_replica_id:9
I161110 14:11:13.598110 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 8 became follower at term 6
I161110 14:11:13.598176 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 8 [peers: [1,2], term: 6, commit: 43, applied: 43, lastindex: 43, lastterm: 6]
I161110 14:11:13.598206 77 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 8 [logterm: 0, index: 45] rejected msgApp [logterm: 6, index: 45] from 1
I161110 14:11:13.598879 452 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.599104 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 43) from 8 for index 45
I161110 14:11:13.599138 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 8 to [next = 44, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.600622 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot adc8c72f at index 47 in 41.268µs.
I161110 14:11:13.601084 274 storage/store.go:2984 [s3,r1/8:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.601636 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 79, log entries: 37
I161110 14:11:13.601659 457 raft/raft.go:568 [s3,r1/8:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:13.601699 457 raft/raft.go:316 [s3,r1/8:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,8], term: 6, commit: 46, applied: 46, lastindex: 47, lastterm: 6]
I161110 14:11:13.601799 457 raft/raft.go:1119 [s3,r1/8:/M{in-ax}] ffffffffffffffff [commit: 46, lastindex: 47, lastterm: 6] fast-forwarded commit to snapshot [index: 47, term: 6]
I161110 14:11:13.601813 457 raft/raft.go:1106 [s3,r1/8:/M{in-ax}] ffffffffffffffff [commit: 47] ignored snapshot [index: 47, term: 6]
I161110 14:11:13.602301 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:9
I161110 14:11:13.602533 201 storage/store.go:2037 [s3,r1/8:/M{in-ax}] DEL 1
I161110 14:11:13.603428 484 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:9}]
I161110 14:11:13.673047 76 raft/raft.go:568 [s3,r1/9:{-}] 9 became follower at term 0
I161110 14:11:13.674014 76 raft/raft.go:316 [s3,r1/9:{-}] newRaft 9 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.674041 76 raft/raft.go:568 [s3,r1/9:{-}] 9 became follower at term 1
I161110 14:11:13.674064 76 raft/raft.go:710 [s3,r1/9:{-}] 9 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:13.674076 76 raft/raft.go:568 [s3,r1/9:{-}] 9 became follower at term 6
I161110 14:11:13.674090 76 raft/raft.go:1088 [s3,r1/9:{-}] 9 [logterm: 0, index: 49] rejected msgApp [logterm: 6, index: 49] from 1
I161110 14:11:13.674366 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 9 for index 49
I161110 14:11:13.674388 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 9 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.674488 54 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 13c2625d at index 49 in 61.759µs.
I161110 14:11:13.674517 54 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 49] sent snapshot[index: 49, term: 6] to 9 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.674532 54 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 9 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 49]
I161110 14:11:13.675547 458 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 82, log entries: 39
I161110 14:11:13.683354 461 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.697407 461 raft/raft.go:1125 [s3,r1/9:{-}] 9 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 49, term: 6]
I161110 14:11:13.697437 461 raft/log.go:288 [s3,r1/9:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 49, term: 6]
I161110 14:11:13.697466 461 raft/raft.go:1135 [s3,r1/9:{-}] 9 restored progress of 1 [next = 50, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.697765 461 raft/raft.go:1135 [s3,r1/9:{-}] 9 restored progress of 2 [next = 50, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.698500 461 raft/raft.go:1135 [s3,r1/9:{-}] 9 restored progress of 9 [next = 50, match = 49, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.698576 461 raft/raft.go:1102 [s3,r1/9:{-}] 9 [commit: 49] restored snapshot [index: 49, term: 6]
I161110 14:11:13.698655 461 storage/replica_raftstorage.go:589 [s3,r1/9:{-}] applying Raft snapshot at index 49 (id=13c2625d, encoded size=16, 1 rocksdb batches, 39 log entries)
I161110 14:11:13.704536 461 storage/replica_raftstorage.go:592 [s3,r1/9:/M{in-ax}] applied Raft snapshot in 0.006s
I161110 14:11:13.710789 55 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 9 [next = 50, match = 49, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 49]
I161110 14:11:13.740352 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:9 > next_replica_id:10
I161110 14:11:13.741706 522 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.745444 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot bb54d260 at index 54 in 39.719µs.
I161110 14:11:13.749054 274 storage/store.go:2984 [s3,r1/9:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.750234 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 88, log entries: 44
I161110 14:11:13.750342 201 storage/store.go:2037 [s3,r1/9:/M{in-ax}] DEL 1
I161110 14:11:13.750508 531 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.750551 531 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.750707 531 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.750722 531 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.750741 531 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.750752 531 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.750862 531 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 54, term: 6]
I161110 14:11:13.750877 531 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 54, term: 6]
I161110 14:11:13.750895 531 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 55, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.750927 531 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 55, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.750939 531 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 54] restored snapshot [index: 54, term: 6]
I161110 14:11:13.750954 531 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 54 (id=bb54d260, encoded size=16, 1 rocksdb batches, 44 log entries)
I161110 14:11:13.752260 531 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161110 14:11:13.752281 531 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.752777 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:10
I161110 14:11:13.753880 500 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:10}]
I161110 14:11:13.755562 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:10 > next_replica_id:11
I161110 14:11:13.756828 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] a became follower at term 6
I161110 14:11:13.756865 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft a [peers: [1,2], term: 6, commit: 54, applied: 54, lastindex: 54, lastterm: 6]
I161110 14:11:13.756906 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] a [logterm: 0, index: 56] rejected msgApp [logterm: 6, index: 56] from 1
I161110 14:11:13.757045 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 54) from a for index 56
I161110 14:11:13.757066 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of a to [next = 55, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.757799 548 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.759633 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 7414d9e9 at index 58 in 41.823µs.
I161110 14:11:13.759849 274 storage/store.go:2984 [s3,r1/10:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.760310 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 94, log entries: 48
I161110 14:11:13.760792 201 storage/store.go:2037 [s3,r1/10:/M{in-ax}] DEL 1
I161110 14:11:13.760952 567 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.760992 567 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.761152 567 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.761165 567 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.761184 567 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.761195 567 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.761311 567 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 58, term: 6]
I161110 14:11:13.761325 567 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 58, term: 6]
I161110 14:11:13.761342 567 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 59, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.761356 567 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 59, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.761368 567 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 58] restored snapshot [index: 58, term: 6]
I161110 14:11:13.761384 567 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 58 (id=7414d9e9, encoded size=16, 1 rocksdb batches, 48 log entries)
I161110 14:11:13.763045 567 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161110 14:11:13.763064 567 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.763513 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:11
I161110 14:11:13.764710 502 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:11}]
I161110 14:11:13.767225 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:11 > next_replica_id:12
I161110 14:11:13.771689 178 raft/raft.go:568 [s3,r1/?:/M{in-ax}] b became follower at term 6
I161110 14:11:13.771801 178 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft b [peers: [1,2], term: 6, commit: 58, applied: 58, lastindex: 58, lastterm: 6]
I161110 14:11:13.771835 178 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] b [logterm: 0, index: 60] rejected msgApp [logterm: 6, index: 60] from 1
I161110 14:11:13.772576 536 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:13.776982 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 8efe6171 at index 62 in 43.472µs.
I161110 14:11:13.777469 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.778006 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 100, log entries: 52
I161110 14:11:13.814011 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:13.901168 201 storage/store.go:2037 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.901597 588 storage/store.go:1908 [s3] ADD 1
I161110 14:11:13.901807 588 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:13.902153 588 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:13.902204 588 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:13.902258 588 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:13.902298 588 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:13.902606 588 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 62, term: 6]
I161110 14:11:13.902634 588 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 62, term: 6]
I161110 14:11:13.902821 588 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 63, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.902906 588 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 63, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:13.902933 588 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 62] restored snapshot [index: 62, term: 6]
I161110 14:11:13.903080 588 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 62 (id=8efe6171, encoded size=16, 1 rocksdb batches, 52 log entries)
I161110 14:11:13.907190 588 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161110 14:11:13.907219 588 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:13.908038 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:12
I161110 14:11:13.909240 599 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:12}]
I161110 14:11:13.971169 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] c became follower at term 6
I161110 14:11:13.983128 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft c [peers: [1,2], term: 6, commit: 62, applied: 62, lastindex: 62, lastterm: 6]
I161110 14:11:13.983978 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:12 > next_replica_id:13
I161110 14:11:13.989961 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] c [logterm: 0, index: 67] rejected msgApp [logterm: 6, index: 67] from 1
I161110 14:11:14.037062 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 62) from c for index 67
I161110 14:11:14.037096 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of c to [next = 63, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.039738 542 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.042416 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 31bd3824 at index 71 in 42.192µs.
I161110 14:11:14.046565 201 storage/store.go:2037 [s3,r1/12:/M{in-ax}] DEL 1
I161110 14:11:14.118297 649 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.118422 649 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.121292 649 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.121590 649 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.121653 649 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.121777 649 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.121998 649 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 71, term: 6]
I161110 14:11:14.122016 649 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 71, term: 6]
I161110 14:11:14.122034 649 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 72, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.122048 649 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 72, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.122066 649 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 71] restored snapshot [index: 71, term: 6]
I161110 14:11:14.122083 649 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 71 (id=31bd3824, encoded size=16, 1 rocksdb batches, 61 log entries)
I161110 14:11:14.124204 649 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161110 14:11:14.124225 649 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.124726 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 111, log entries: 61
I161110 14:11:14.172162 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:13
I161110 14:11:14.206428 633 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:13}]
I161110 14:11:14.229062 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:13 > next_replica_id:14
I161110 14:11:14.230693 79 raft/raft.go:568 [s3,r1/?:/M{in-ax}] d became follower at term 6
I161110 14:11:14.230728 79 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft d [peers: [1,2], term: 6, commit: 71, applied: 71, lastindex: 71, lastterm: 6]
I161110 14:11:14.230768 79 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] d [logterm: 0, index: 76] rejected msgApp [logterm: 6, index: 76] from 1
I161110 14:11:14.230795 79 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] d [logterm: 0, index: 76] rejected msgApp [logterm: 6, index: 76] from 1
I161110 14:11:14.230922 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 71) from d for index 76
I161110 14:11:14.230944 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of d to [next = 72, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.230992 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 71) from d for index 76
I161110 14:11:14.233097 682 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.235221 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 9b955fbd at index 79 in 34.649µs.
I161110 14:11:14.237427 201 storage/store.go:2037 [s3,r1/13:/M{in-ax}] DEL 1
I161110 14:11:14.238145 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 121, log entries: 69
I161110 14:11:14.238422 690 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.238470 690 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.238718 690 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.238731 690 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.238753 690 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.238764 690 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.238935 690 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 79, term: 6]
I161110 14:11:14.238949 690 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 79, term: 6]
I161110 14:11:14.238966 690 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 80, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.238979 690 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 80, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.238990 690 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 79] restored snapshot [index: 79, term: 6]
I161110 14:11:14.239006 690 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 79 (id=9b955fbd, encoded size=16, 1 rocksdb batches, 69 log entries)
I161110 14:11:14.240869 690 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161110 14:11:14.240894 690 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.241404 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:14
I161110 14:11:14.241954 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.242366 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.243907 201 storage/store.go:2037 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.244578 666 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:14}]
I161110 14:11:14.358823 75 raft/raft.go:568 [s3,r1/14:{-}] e became follower at term 0
I161110 14:11:14.362324 75 raft/raft.go:316 [s3,r1/14:{-}] newRaft e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.362364 75 raft/raft.go:568 [s3,r1/14:{-}] e became follower at term 1
I161110 14:11:14.362410 75 raft/raft.go:710 [s3,r1/14:{-}] e [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161110 14:11:14.362516 75 raft/raft.go:568 [s3,r1/14:{-}] e became follower at term 6
I161110 14:11:14.363321 80 raft/raft.go:1088 [s3,r1/14:{-}] e [logterm: 0, index: 81] rejected msgApp [logterm: 6, index: 81] from 1
I161110 14:11:14.363461 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from e for index 81
I161110 14:11:14.363506 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of e to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.363745 54 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 5fb7c3e7 at index 81 in 162.079µs.
I161110 14:11:14.363784 54 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 81] sent snapshot[index: 81, term: 6] to e [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.363813 54 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to e [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 81]
I161110 14:11:14.366020 619 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.366544 619 raft/raft.go:1125 [s3,r1/14:{-}] e [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 81, term: 6]
I161110 14:11:14.366577 619 raft/log.go:288 [s3,r1/14:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 81, term: 6]
I161110 14:11:14.366607 619 raft/raft.go:1135 [s3,r1/14:{-}] e restored progress of 1 [next = 82, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.366632 619 raft/raft.go:1135 [s3,r1/14:{-}] e restored progress of 2 [next = 82, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.366794 619 raft/raft.go:1135 [s3,r1/14:{-}] e restored progress of e [next = 82, match = 81, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.367015 619 raft/raft.go:1102 [s3,r1/14:{-}] e [commit: 81] restored snapshot [index: 81, term: 6]
I161110 14:11:14.367143 619 storage/replica_raftstorage.go:589 [s3,r1/14:{-}] applying Raft snapshot at index 81 (id=5fb7c3e7, encoded size=16, 1 rocksdb batches, 71 log entries)
I161110 14:11:14.368344 667 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 124, log entries: 71
I161110 14:11:14.370661 619 storage/replica_raftstorage.go:592 [s3,r1/14:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:14.410387 59 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to e [next = 82, match = 81, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 81]
I161110 14:11:14.428427 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:14 > next_replica_id:15
I161110 14:11:14.429664 510 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.431202 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 08edd651 at index 86 in 37.484µs.
I161110 14:11:14.432432 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 130, log entries: 76
I161110 14:11:14.432585 274 storage/store.go:2984 [s3,r1/14:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.433306 201 storage/store.go:2037 [s3,r1/14:/M{in-ax}] DEL 1
I161110 14:11:14.434217 743 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.434345 743 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.434638 743 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.434653 743 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.434673 743 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.434685 743 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.434916 743 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 86, term: 6]
I161110 14:11:14.434930 743 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 86, term: 6]
I161110 14:11:14.434974 743 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 87, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.434988 743 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 87, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.434999 743 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 86] restored snapshot [index: 86, term: 6]
I161110 14:11:14.435014 743 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 86 (id=08edd651, encoded size=16, 1 rocksdb batches, 76 log entries)
I161110 14:11:14.437048 743 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161110 14:11:14.437073 743 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.437664 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:15
I161110 14:11:14.439036 730 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:15}]
I161110 14:11:14.440690 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:15 > next_replica_id:16
I161110 14:11:14.444636 711 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.444807 81 raft/raft.go:568 [s3,r1/?:/M{in-ax}] f became follower at term 6
I161110 14:11:14.444842 81 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft f [peers: [1,2], term: 6, commit: 86, applied: 86, lastindex: 86, lastterm: 6]
I161110 14:11:14.444859 81 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] f [logterm: 0, index: 88] rejected msgApp [logterm: 6, index: 88] from 1
I161110 14:11:14.445078 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 86) from f for index 88
I161110 14:11:14.445123 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of f to [next = 87, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.448389 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot b5bcf173 at index 90 in 44.439µs.
I161110 14:11:14.452424 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 136, log entries: 80
I161110 14:11:14.452512 776 raft/raft.go:568 [s3,r1/15:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:14.452542 776 raft/raft.go:316 [s3,r1/15:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 90, applied: 90, lastindex: 90, lastterm: 6]
I161110 14:11:14.452582 776 raft/raft.go:1106 [s3,r1/15:/M{in-ax}] ffffffffffffffff [commit: 90] ignored snapshot [index: 90, term: 6]
I161110 14:11:14.470624 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:16
I161110 14:11:14.474262 715 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:16}]
I161110 14:11:14.488267 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:16 > next_replica_id:17
I161110 14:11:14.494451 80 raft/raft.go:568 [s3,r1/15:/M{in-ax}] 10 became follower at term 6
I161110 14:11:14.494494 80 raft/raft.go:316 [s3,r1/15:/M{in-ax}] newRaft 10 [peers: [1,2], term: 6, commit: 90, applied: 90, lastindex: 90, lastterm: 6]
I161110 14:11:14.494529 80 raft/raft.go:1088 [s3,r1/15:/M{in-ax}] 10 [logterm: 0, index: 92] rejected msgApp [logterm: 6, index: 92] from 1
I161110 14:11:14.494722 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 90) from 10 for index 92
I161110 14:11:14.494748 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 10 to [next = 91, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.532578 790 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.547100 274 storage/store.go:2984 [s3,r1/16:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.547719 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 15e038ff at index 94 in 141.852µs.
I161110 14:11:14.548351 201 storage/store.go:2037 [s3,r1/16:/M{in-ax}] DEL 1
I161110 14:11:14.549179 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 142, log entries: 84
I161110 14:11:14.549560 795 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.549647 795 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.550158 795 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.550229 795 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.550267 795 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.550317 795 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.550693 795 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 94, term: 6]
I161110 14:11:14.550730 795 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 94, term: 6]
I161110 14:11:14.550761 795 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 95, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.550787 795 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 95, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.550813 795 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 94] restored snapshot [index: 94, term: 6]
I161110 14:11:14.550867 795 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 94 (id=15e038ff, encoded size=16, 1 rocksdb batches, 84 log entries)
I161110 14:11:14.578153 795 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.027s
I161110 14:11:14.584143 795 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.606832 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:17
I161110 14:11:14.607022 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.627452 201 storage/store.go:2037 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.722020 819 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:17}]
I161110 14:11:14.747432 77 raft/raft.go:568 [s3,r1/17:{-}] 11 became follower at term 0
I161110 14:11:14.747746 77 raft/raft.go:316 [s3,r1/17:{-}] newRaft 11 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.747762 77 raft/raft.go:568 [s3,r1/17:{-}] 11 became follower at term 1
I161110 14:11:14.747784 77 raft/raft.go:710 [s3,r1/17:{-}] 11 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:14.747795 77 raft/raft.go:568 [s3,r1/17:{-}] 11 became follower at term 6
I161110 14:11:14.747809 77 raft/raft.go:1088 [s3,r1/17:{-}] 11 [logterm: 0, index: 101] rejected msgApp [logterm: 6, index: 101] from 1
I161110 14:11:14.748405 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 11 for index 101
I161110 14:11:14.748427 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 11 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.748510 58 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 8ef0ba1e at index 101 in 47.63µs.
I161110 14:11:14.748537 58 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 101] sent snapshot[index: 101, term: 6] to 11 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.748553 58 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 11 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 101]
I161110 14:11:14.749826 813 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 150, log entries: 91
I161110 14:11:14.749905 836 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.750248 836 raft/raft.go:1125 [s3,r1/17:{-}] 11 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 101, term: 6]
I161110 14:11:14.750264 836 raft/log.go:288 [s3,r1/17:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 101, term: 6]
I161110 14:11:14.750280 836 raft/raft.go:1135 [s3,r1/17:{-}] 11 restored progress of 1 [next = 102, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.750293 836 raft/raft.go:1135 [s3,r1/17:{-}] 11 restored progress of 2 [next = 102, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.750305 836 raft/raft.go:1135 [s3,r1/17:{-}] 11 restored progress of 11 [next = 102, match = 101, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.750315 836 raft/raft.go:1102 [s3,r1/17:{-}] 11 [commit: 101] restored snapshot [index: 101, term: 6]
I161110 14:11:14.750336 836 storage/replica_raftstorage.go:589 [s3,r1/17:{-}] applying Raft snapshot at index 101 (id=8ef0ba1e, encoded size=16, 1 rocksdb batches, 91 log entries)
I161110 14:11:14.752913 836 storage/replica_raftstorage.go:592 [s3,r1/17:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:14.753184 57 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 11 [next = 102, match = 101, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 101]
I161110 14:11:14.781392 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:17 > next_replica_id:18
I161110 14:11:14.783221 851 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.787788 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 3ae44b6a at index 104 in 82.316µs.
I161110 14:11:14.796472 274 storage/store.go:2984 [s3,r1/17:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.798483 201 storage/store.go:2037 [s3,r1/17:/M{in-ax}] DEL 1
I161110 14:11:14.799574 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 154, log entries: 94
I161110 14:11:14.799786 858 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.799853 858 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.800219 858 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.800236 858 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.800256 858 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.800268 858 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.800604 858 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 104, term: 6]
I161110 14:11:14.800625 858 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 104, term: 6]
I161110 14:11:14.800644 858 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 105, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.800659 858 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 105, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.800670 858 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 104] restored snapshot [index: 104, term: 6]
I161110 14:11:14.800685 858 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 104 (id=3ae44b6a, encoded size=16, 1 rocksdb batches, 94 log entries)
I161110 14:11:14.803534 858 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161110 14:11:14.803557 858 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.804064 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:18
I161110 14:11:14.805184 841 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:18}]
I161110 14:11:14.806785 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:18 > next_replica_id:19
I161110 14:11:14.807607 81 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 12 became follower at term 6
I161110 14:11:14.807643 81 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 12 [peers: [1,2], term: 6, commit: 104, applied: 104, lastindex: 104, lastterm: 6]
I161110 14:11:14.807660 81 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 12 [logterm: 0, index: 106] rejected msgApp [logterm: 6, index: 106] from 1
I161110 14:11:14.807776 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 104) from 12 for index 106
I161110 14:11:14.807798 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 12 to [next = 105, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.808519 885 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.812302 201 storage/store.go:2037 [s3,r1/18:/M{in-ax}] DEL 1
I161110 14:11:14.813224 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 4d6c8bb3 at index 108 in 40.717µs.
I161110 14:11:14.814421 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 160, log entries: 98
I161110 14:11:14.814716 892 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.817364 892 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.817982 892 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.818012 892 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.818055 892 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.818068 892 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.818919 892 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 108, term: 6]
I161110 14:11:14.818951 892 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 108, term: 6]
I161110 14:11:14.821845 892 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 109, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.821874 892 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 109, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.821893 892 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 108] restored snapshot [index: 108, term: 6]
I161110 14:11:14.821979 892 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 108 (id=4d6c8bb3, encoded size=16, 1 rocksdb batches, 98 log entries)
I161110 14:11:14.827008 892 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161110 14:11:14.827033 892 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.827548 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:19
I161110 14:11:14.834584 761 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:19}]
I161110 14:11:14.841075 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:19 > next_replica_id:20
I161110 14:11:14.844660 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 13 became follower at term 6
I161110 14:11:14.844701 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 13 [peers: [1,2], term: 6, commit: 108, applied: 108, lastindex: 108, lastterm: 6]
I161110 14:11:14.844744 75 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 13 [logterm: 0, index: 110] rejected msgApp [logterm: 6, index: 110] from 1
I161110 14:11:14.845135 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 108) from 13 for index 110
I161110 14:11:14.845159 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 13 to [next = 109, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.846247 893 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.847258 274 storage/store.go:2984 [s3,r1/19:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:14.849066 201 storage/store.go:2037 [s3,r1/19:/M{in-ax}] DEL 1
I161110 14:11:14.849381 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 895faea3 at index 112 in 48.399µs.
I161110 14:11:14.860455 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 166, log entries: 102
I161110 14:11:14.860679 879 storage/store.go:1908 [s3] ADD 1
I161110 14:11:14.860725 879 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:14.861125 879 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:14.861140 879 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:14.861161 879 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:14.861173 879 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:14.861481 879 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 112, term: 6]
I161110 14:11:14.861498 879 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 112, term: 6]
I161110 14:11:14.861515 879 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.861529 879 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.861539 879 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 112] restored snapshot [index: 112, term: 6]
I161110 14:11:14.861554 879 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 112 (id=895faea3, encoded size=16, 1 rocksdb batches, 102 log entries)
I161110 14:11:14.864243 879 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161110 14:11:14.864263 879 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:14.864810 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:20
I161110 14:11:14.866291 919 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:20}]
I161110 14:11:14.868118 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:20 > next_replica_id:21
I161110 14:11:14.870023 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 14 became follower at term 6
I161110 14:11:14.870061 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 14 [peers: [1,2], term: 6, commit: 112, applied: 112, lastindex: 112, lastterm: 6]
I161110 14:11:14.870090 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 14 [logterm: 0, index: 114] rejected msgApp [logterm: 6, index: 114] from 1
I161110 14:11:14.870229 59 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 112) from 14 for index 114
I161110 14:11:14.870263 59 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 14 to [next = 113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.871387 880 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.872985 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 1e819bd5 at index 116 in 39.633µs.
I161110 14:11:14.874188 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 172, log entries: 106
I161110 14:11:14.874316 904 raft/raft.go:568 [s3,r1/20:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:14.874352 904 raft/raft.go:316 [s3,r1/20:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 116, applied: 116, lastindex: 116, lastterm: 6]
I161110 14:11:14.874405 904 raft/raft.go:1106 [s3,r1/20:/M{in-ax}] ffffffffffffffff [commit: 116] ignored snapshot [index: 116, term: 6]
I161110 14:11:14.874911 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:21
I161110 14:11:14.876237 906 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:21}]
I161110 14:11:14.877845 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:21 > next_replica_id:22
I161110 14:11:14.878539 80 raft/raft.go:568 [s3,r1/20:/M{in-ax}] 15 became follower at term 6
I161110 14:11:14.878576 80 raft/raft.go:316 [s3,r1/20:/M{in-ax}] newRaft 15 [peers: [1,2], term: 6, commit: 116, applied: 116, lastindex: 116, lastterm: 6]
I161110 14:11:14.878600 80 raft/raft.go:1088 [s3,r1/20:/M{in-ax}] 15 [logterm: 0, index: 118] rejected msgApp [logterm: 6, index: 118] from 1
I161110 14:11:14.878936 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 116) from 15 for index 118
I161110 14:11:14.878958 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 15 to [next = 117, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.879375 928 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.881239 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 8ae98c04 at index 120 in 48.512µs.
I161110 14:11:14.882347 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 178, log entries: 110
I161110 14:11:14.882508 909 raft/raft.go:568 [s3,r1/21:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:14.882543 909 raft/raft.go:316 [s3,r1/21:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 120, applied: 120, lastindex: 120, lastterm: 6]
I161110 14:11:14.882588 909 raft/raft.go:1106 [s3,r1/21:/M{in-ax}] ffffffffffffffff [commit: 120] ignored snapshot [index: 120, term: 6]
I161110 14:11:14.883071 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:22
I161110 14:11:14.884304 953 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:22}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:22}]
I161110 14:11:14.885679 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:22 > next_replica_id:23
I161110 14:11:14.890888 80 raft/raft.go:568 [s3,r1/21:/M{in-ax}] 16 became follower at term 6
I161110 14:11:14.890947 80 raft/raft.go:316 [s3,r1/21:/M{in-ax}] newRaft 16 [peers: [1,2], term: 6, commit: 120, applied: 120, lastindex: 120, lastterm: 6]
I161110 14:11:14.890994 80 raft/raft.go:1088 [s3,r1/21:/M{in-ax}] 16 [logterm: 0, index: 122] rejected msgApp [logterm: 6, index: 122] from 1
I161110 14:11:14.891204 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 120) from 16 for index 122
I161110 14:11:14.891242 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 16 to [next = 121, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:14.894004 985 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:22}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.895900 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 6d20373c at index 124 in 42.572µs.
I161110 14:11:14.897952 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 184, log entries: 114
I161110 14:11:14.898115 975 raft/raft.go:568 [s3,r1/22:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:14.898150 975 raft/raft.go:316 [s3,r1/22:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 124, applied: 124, lastindex: 124, lastterm: 6]
I161110 14:11:14.898196 975 raft/raft.go:1106 [s3,r1/22:/M{in-ax}] ffffffffffffffff [commit: 124] ignored snapshot [index: 124, term: 6]
I161110 14:11:14.898739 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:23
I161110 14:11:14.899960 987 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:23}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:23}]
I161110 14:11:14.901567 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:23 > next_replica_id:24
I161110 14:11:14.902957 80 raft/raft.go:568 [s3,r1/22:/M{in-ax}] 17 became follower at term 6
I161110 14:11:14.902992 80 raft/raft.go:316 [s3,r1/22:/M{in-ax}] newRaft 17 [peers: [1,2], term: 6, commit: 124, applied: 124, lastindex: 124, lastterm: 6]
I161110 14:11:14.903009 80 raft/raft.go:1088 [s3,r1/22:/M{in-ax}] 17 [logterm: 0, index: 126] rejected msgApp [logterm: 6, index: 126] from 1
I161110 14:11:14.903798 977 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:23}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:14.903940 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 124) from 17 for index 126
I161110 14:11:14.903960 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 17 to [next = 125, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.078244 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 0b520f68 at index 131 in 38.08µs.
I161110 14:11:15.078540 91 storage/raft_log_queue.go:219 [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 11-131
I161110 14:11:15.080327 274 storage/store.go:2984 [s3,r1/23:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.082307 201 storage/store.go:2037 [s3,r1/23:/M{in-ax}] DEL 1
I161110 14:11:15.109371 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 193, log entries: 121
I161110 14:11:15.110717 1022 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.111483 1022 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:15.113565 1022 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
E161110 14:11:15.124057 274 storage/store.go:2982 [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:15.124103 1022 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:15.124368 1022 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:15.124444 1022 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:15.126471 1022 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 131, term: 6]
I161110 14:11:15.126572 1022 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 131, term: 6]
I161110 14:11:15.126902 1022 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 132, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.127305 1022 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 132, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.129496 1022 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 131] restored snapshot [index: 131, term: 6]
I161110 14:11:15.132650 1022 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 131 (id=0b520f68, encoded size=16, 1 rocksdb batches, 121 log entries)
I161110 14:11:15.253929 1022 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.121s
I161110 14:11:15.253965 1022 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:15.255237 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:24
I161110 14:11:15.261963 1040 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:24}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:24}]
I161110 14:11:15.296134 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 18 became follower at term 6
I161110 14:11:15.296324 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 18 [peers: [1,2], term: 6, commit: 131, applied: 131, lastindex: 131, lastterm: 6]
I161110 14:11:15.296342 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 18 [logterm: 0, index: 140] rejected msgApp [logterm: 6, index: 140] from 1
I161110 14:11:15.296364 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 18 [logterm: 0, index: 140] rejected msgApp [logterm: 6, index: 140] from 1
I161110 14:11:15.296383 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 18 [logterm: 0, index: 140] rejected msgApp [logterm: 6, index: 140] from 1
I161110 14:11:15.296575 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 131) from 18 for index 140
I161110 14:11:15.296606 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 18 to [next = 132, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.296823 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 131) from 18 for index 140
I161110 14:11:15.296855 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 131) from 18 for index 140
I161110 14:11:15.297027 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:24 > next_replica_id:25
I161110 14:11:15.298883 989 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:24}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.301839 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 87835c99 at index 142 in 36.462µs.
I161110 14:11:15.303124 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 205, log entries: 12
I161110 14:11:15.303399 1094 raft/raft.go:568 [s3,r1/24:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:15.303432 1094 raft/raft.go:316 [s3,r1/24:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 142, applied: 142, lastindex: 142, lastterm: 6]
I161110 14:11:15.303474 1094 raft/raft.go:1106 [s3,r1/24:/M{in-ax}] ffffffffffffffff [commit: 142] ignored snapshot [index: 142, term: 6]
I161110 14:11:15.303904 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:25
I161110 14:11:15.305029 1098 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:25}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:25}]
I161110 14:11:15.306739 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:25 > next_replica_id:26
I161110 14:11:15.316636 201 storage/store.go:2037 [s3,r1/24:/M{in-ax}] DEL 1
I161110 14:11:15.319716 80 raft/raft.go:568 [s3,r1/25:{-}] 19 became follower at term 0
I161110 14:11:15.320170 80 raft/raft.go:316 [s3,r1/25:{-}] newRaft 19 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:15.320188 80 raft/raft.go:568 [s3,r1/25:{-}] 19 became follower at term 1
I161110 14:11:15.320220 80 raft/raft.go:710 [s3,r1/25:{-}] 19 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:15.320235 80 raft/raft.go:568 [s3,r1/25:{-}] 19 became follower at term 6
I161110 14:11:15.320248 80 raft/raft.go:1088 [s3,r1/25:{-}] 19 [logterm: 0, index: 144] rejected msgApp [logterm: 6, index: 144] from 1
I161110 14:11:15.320466 78 raft/raft.go:1088 [s3,r1/25:{-}] 19 [logterm: 0, index: 144] rejected msgApp [logterm: 6, index: 144] from 1
I161110 14:11:15.321205 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 19 for index 144
I161110 14:11:15.321287 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 19 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.321416 55 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 873499bb at index 145 in 61.111µs.
I161110 14:11:15.321479 55 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 131, commit: 145] sent snapshot[index: 145, term: 6] to 19 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.321506 55 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 19 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 145]
I161110 14:11:15.321594 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 19 for index 144
I161110 14:11:15.322184 1122 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:25}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.324498 1110 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.324583 1078 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 211, log entries: 15
I161110 14:11:15.325636 1110 raft/raft.go:1125 [s3,r1/25:{-}] 19 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 145, term: 6]
I161110 14:11:15.325655 1110 raft/log.go:288 [s3,r1/25:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 145, term: 6]
I161110 14:11:15.325672 1110 raft/raft.go:1135 [s3,r1/25:{-}] 19 restored progress of 1 [next = 146, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.325699 1110 raft/raft.go:1135 [s3,r1/25:{-}] 19 restored progress of 2 [next = 146, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.325727 1110 raft/raft.go:1135 [s3,r1/25:{-}] 19 restored progress of 19 [next = 146, match = 145, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.325745 1110 raft/raft.go:1102 [s3,r1/25:{-}] 19 [commit: 145] restored snapshot [index: 145, term: 6]
I161110 14:11:15.325772 1110 storage/replica_raftstorage.go:589 [s3,r1/25:{-}] applying Raft snapshot at index 145 (id=873499bb, encoded size=16, 1 rocksdb batches, 15 log entries)
I161110 14:11:15.327736 1110 storage/replica_raftstorage.go:592 [s3,r1/25:/M{in-ax}] applied Raft snapshot in 0.002s
I161110 14:11:15.327970 1078 raft/raft.go:854 [s1,r1/1:/M{in-ax}] 1 no progress available for 19
I161110 14:11:15.328109 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 5d484290 at index 146 in 70.514µs.
I161110 14:11:15.328312 274 storage/store.go:2984 [s3,r1/25:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.328802 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 211, log entries: 16
I161110 14:11:15.347594 201 storage/store.go:2037 [s3,r1/25:/M{in-ax}] DEL 1
I161110 14:11:15.348033 1084 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.348115 1084 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:15.357541 1084 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:15.357619 1084 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:15.357652 1084 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:15.357665 1084 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:15.358931 1084 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 146, term: 6]
I161110 14:11:15.359035 1084 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 146, term: 6]
I161110 14:11:15.359066 1084 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 147, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.359095 1084 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 147, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.359108 1084 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 146] restored snapshot [index: 146, term: 6]
I161110 14:11:15.359124 1084 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 146 (id=5d484290, encoded size=16, 1 rocksdb batches, 16 log entries)
I161110 14:11:15.371953 1084 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.013s
I161110 14:11:15.372000 1084 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:15.372219 79 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 19 became follower at term 6
I161110 14:11:15.372335 79 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 19 [peers: [1,2], term: 6, commit: 146, applied: 146, lastindex: 146, lastterm: 6]
I161110 14:11:15.372666 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:26
I161110 14:11:15.373888 1088 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:26}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:26}]
I161110 14:11:15.378462 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.379342 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:26 > next_replica_id:27
I161110 14:11:15.467410 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 1a became follower at term 6
I161110 14:11:15.467445 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 1a [peers: [1,2], term: 6, commit: 146, applied: 146, lastindex: 146, lastterm: 6]
I161110 14:11:15.504290 1145 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:26}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.507485 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot bcc02097 at index 152 in 36.111µs.
I161110 14:11:15.521609 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 219, log entries: 22
I161110 14:11:15.521851 1176 raft/raft.go:568 [s3,r1/?:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:15.521883 1176 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 146, applied: 146, lastindex: 146, lastterm: 6]
I161110 14:11:15.521937 1176 raft/raft.go:1125 [s3,r1/?:/M{in-ax}] ffffffffffffffff [commit: 146, lastindex: 146, lastterm: 6] starts to restore snapshot [index: 152, term: 6]
I161110 14:11:15.521951 1176 raft/log.go:288 [s3,r1/?:/M{in-ax}] log [committed=146, applied=146, unstable.offset=147, len(unstable.Entries)=0] starts to restore snapshot [index: 152, term: 6]
I161110 14:11:15.522082 1176 raft/raft.go:1135 [s3,r1/?:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 153, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.522608 1176 raft/raft.go:1135 [s3,r1/?:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 153, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.522623 1176 raft/raft.go:1102 [s3,r1/?:/M{in-ax}] ffffffffffffffff [commit: 152] restored snapshot [index: 152, term: 6]
I161110 14:11:15.522640 1176 storage/replica_raftstorage.go:589 [s3,r1/?:/M{in-ax}] applying Raft snapshot at index 152 (id=bcc02097, encoded size=16, 1 rocksdb batches, 22 log entries)
I161110 14:11:15.553425 1176 storage/replica_raftstorage.go:592 [s3,r1/26:/M{in-ax}] applied Raft snapshot in 0.031s
I161110 14:11:15.553685 78 raft/raft.go:568 [s3,r1/26:/M{in-ax}] 1a became follower at term 6
I161110 14:11:15.553774 78 raft/raft.go:316 [s3,r1/26:/M{in-ax}] newRaft 1a [peers: [1,2], term: 6, commit: 152, applied: 152, lastindex: 152, lastterm: 6]
I161110 14:11:15.553977 274 storage/store.go:2984 [s3,r1/26:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.555862 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:27
I161110 14:11:15.556029 201 storage/store.go:2037 [s3,r1/26:/M{in-ax}] DEL 1
I161110 14:11:15.557165 1120 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:27}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:27}]
I161110 14:11:15.648157 76 raft/raft.go:568 [s3,r1/27:{-}] 1b became follower at term 0
I161110 14:11:15.648779 76 raft/raft.go:316 [s3,r1/27:{-}] newRaft 1b [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:15.648818 76 raft/raft.go:568 [s3,r1/27:{-}] 1b became follower at term 1
I161110 14:11:15.648888 76 raft/raft.go:710 [s3,r1/27:{-}] 1b [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:15.648909 76 raft/raft.go:568 [s3,r1/27:{-}] 1b became follower at term 6
I161110 14:11:15.648927 76 raft/raft.go:1088 [s3,r1/27:{-}] 1b [logterm: 0, index: 155] rejected msgApp [logterm: 6, index: 155] from 1
I161110 14:11:15.649160 53 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 1b for index 155
I161110 14:11:15.649185 53 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 1b to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.649265 53 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 82bafc40 at index 157 in 43.563µs.
I161110 14:11:15.649293 53 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 131, commit: 157] sent snapshot[index: 157, term: 6] to 1b [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.649309 53 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 1b [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 157]
I161110 14:11:15.650527 1166 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 225, log entries: 27
I161110 14:11:15.650648 1134 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.651153 1134 raft/raft.go:1125 [s3,r1/27:{-}] 1b [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 157, term: 6]
I161110 14:11:15.651177 1134 raft/log.go:288 [s3,r1/27:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 157, term: 6]
I161110 14:11:15.651198 1134 raft/raft.go:1135 [s3,r1/27:{-}] 1b restored progress of 1 [next = 158, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.651220 1134 raft/raft.go:1135 [s3,r1/27:{-}] 1b restored progress of 2 [next = 158, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.651235 1134 raft/raft.go:1135 [s3,r1/27:{-}] 1b restored progress of 1b [next = 158, match = 157, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.651257 1134 raft/raft.go:1102 [s3,r1/27:{-}] 1b [commit: 157] restored snapshot [index: 157, term: 6]
I161110 14:11:15.651295 1134 storage/replica_raftstorage.go:589 [s3,r1/27:{-}] applying Raft snapshot at index 157 (id=82bafc40, encoded size=16, 1 rocksdb batches, 27 log entries)
I161110 14:11:15.653551 1134 storage/replica_raftstorage.go:592 [s3,r1/27:/M{in-ax}] applied Raft snapshot in 0.002s
I161110 14:11:15.653855 52 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 1b [next = 158, match = 157, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 157]
I161110 14:11:15.668009 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:27 > next_replica_id:28
I161110 14:11:15.669813 1193 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:27}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.671209 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 9a08cc38 at index 160 in 40.04µs.
I161110 14:11:15.672385 274 storage/store.go:2984 [s3,r1/27:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.672983 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 229, log entries: 30
I161110 14:11:15.675059 201 storage/store.go:2037 [s3,r1/27:/M{in-ax}] DEL 1
I161110 14:11:15.676006 1194 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.676116 1194 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:15.676694 1194 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:15.676711 1194 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:15.676754 1194 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:15.676771 1194 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:15.677462 1194 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 160, term: 6]
I161110 14:11:15.677482 1194 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 160, term: 6]
I161110 14:11:15.677502 1194 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 161, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.677517 1194 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 161, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.677528 1194 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 160] restored snapshot [index: 160, term: 6]
I161110 14:11:15.677545 1194 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 160 (id=9a08cc38, encoded size=16, 1 rocksdb batches, 30 log entries)
I161110 14:11:15.680042 1194 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161110 14:11:15.680059 1194 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:15.680172 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.680561 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:28
I161110 14:11:15.682271 1198 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:28}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:28}]
I161110 14:11:15.684046 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:28 > next_replica_id:29
I161110 14:11:15.685524 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 1c became follower at term 6
I161110 14:11:15.685561 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 1c [peers: [1,2], term: 6, commit: 160, applied: 160, lastindex: 160, lastterm: 6]
I161110 14:11:15.685579 75 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 1c [logterm: 0, index: 162] rejected msgApp [logterm: 6, index: 162] from 1
I161110 14:11:15.685698 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 160) from 1c for index 162
I161110 14:11:15.685721 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 1c to [next = 161, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.686590 1242 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:28}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.689112 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 0dd00067 at index 164 in 38.109µs.
I161110 14:11:15.694793 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 235, log entries: 34
I161110 14:11:15.696150 1253 raft/raft.go:568 [s3,r1/28:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:15.696192 1253 raft/raft.go:316 [s3,r1/28:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 164, applied: 164, lastindex: 164, lastterm: 6]
I161110 14:11:15.696262 1253 raft/raft.go:1106 [s3,r1/28:/M{in-ax}] ffffffffffffffff [commit: 164] ignored snapshot [index: 164, term: 6]
I161110 14:11:15.696929 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:29
I161110 14:11:15.700851 1208 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:29}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:29}]
I161110 14:11:15.746989 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:29 > next_replica_id:30
I161110 14:11:15.748694 81 raft/raft.go:568 [s3,r1/28:/M{in-ax}] 1d became follower at term 6
I161110 14:11:15.748734 81 raft/raft.go:316 [s3,r1/28:/M{in-ax}] newRaft 1d [peers: [1,2], term: 6, commit: 164, applied: 164, lastindex: 164, lastterm: 6]
I161110 14:11:15.748751 81 raft/raft.go:1088 [s3,r1/28:/M{in-ax}] 1d [logterm: 0, index: 166] rejected msgApp [logterm: 6, index: 166] from 1
I161110 14:11:15.751625 1265 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:29}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.751822 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 164) from 1d for index 166
I161110 14:11:15.751848 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 1d to [next = 165, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.757116 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot d0ad1ed8 at index 168 in 182.978µs.
I161110 14:11:15.775609 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 241, log entries: 38
I161110 14:11:15.776722 1276 raft/raft.go:568 [s3,r1/29:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:15.777123 1276 raft/raft.go:316 [s3,r1/29:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,1d], term: 6, commit: 167, applied: 167, lastindex: 168, lastterm: 6]
I161110 14:11:15.777270 1276 raft/raft.go:1119 [s3,r1/29:/M{in-ax}] ffffffffffffffff [commit: 167, lastindex: 168, lastterm: 6] fast-forwarded commit to snapshot [index: 168, term: 6]
I161110 14:11:15.777292 1276 raft/raft.go:1106 [s3,r1/29:/M{in-ax}] ffffffffffffffff [commit: 168] ignored snapshot [index: 168, term: 6]
I161110 14:11:15.777866 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:30
I161110 14:11:15.789799 201 storage/store.go:2037 [s3,r1/29:/M{in-ax}] DEL 1
I161110 14:11:15.792364 1279 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:30}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:30}]
I161110 14:11:15.838136 77 raft/raft.go:568 [s3,r1/30:{-}] 1e became follower at term 0
I161110 14:11:15.838706 77 raft/raft.go:316 [s3,r1/30:{-}] newRaft 1e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:15.838721 77 raft/raft.go:568 [s3,r1/30:{-}] 1e became follower at term 1
I161110 14:11:15.838741 77 raft/raft.go:710 [s3,r1/30:{-}] 1e [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161110 14:11:15.838750 77 raft/raft.go:568 [s3,r1/30:{-}] 1e became follower at term 6
I161110 14:11:15.838776 77 raft/raft.go:1088 [s3,r1/30:{-}] 1e [logterm: 0, index: 170] rejected msgApp [logterm: 6, index: 170] from 1
I161110 14:11:15.838958 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 1e for index 170
I161110 14:11:15.838978 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 1e to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.839062 55 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 85b8268d at index 170 in 50.106µs.
I161110 14:11:15.839087 55 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 131, commit: 170] sent snapshot[index: 170, term: 6] to 1e [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.839104 55 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 1e [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 170]
I161110 14:11:15.860833 1300 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 244, log entries: 40
I161110 14:11:15.860976 1304 storage/store.go:1908 [s3] ADD 1
I161110 14:11:15.861624 1304 raft/raft.go:1125 [s3,r1/30:{-}] 1e [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 170, term: 6]
I161110 14:11:15.861655 1304 raft/log.go:288 [s3,r1/30:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 170, term: 6]
I161110 14:11:15.861673 1304 raft/raft.go:1135 [s3,r1/30:{-}] 1e restored progress of 1 [next = 171, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.861688 1304 raft/raft.go:1135 [s3,r1/30:{-}] 1e restored progress of 2 [next = 171, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.861701 1304 raft/raft.go:1135 [s3,r1/30:{-}] 1e restored progress of 1e [next = 171, match = 170, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.861712 1304 raft/raft.go:1102 [s3,r1/30:{-}] 1e [commit: 170] restored snapshot [index: 170, term: 6]
I161110 14:11:15.861736 1304 storage/replica_raftstorage.go:589 [s3,r1/30:{-}] applying Raft snapshot at index 170 (id=85b8268d, encoded size=16, 1 rocksdb batches, 40 log entries)
I161110 14:11:15.864394 1304 storage/replica_raftstorage.go:592 [s3,r1/30:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:15.867830 59 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 1e [next = 171, match = 170, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 170]
I161110 14:11:15.918350 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:30 > next_replica_id:31
I161110 14:11:15.924736 1309 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:30}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:15.928368 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 3ee80bbe at index 175 in 41.715µs.
I161110 14:11:15.930132 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 250, log entries: 45
I161110 14:11:15.930323 1325 raft/raft.go:568 [s3,r1/30:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:15.930356 1325 raft/raft.go:316 [s3,r1/30:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,1e], term: 6, commit: 172, applied: 172, lastindex: 174, lastterm: 6]
I161110 14:11:15.930408 1325 raft/raft.go:1125 [s3,r1/30:/M{in-ax}] ffffffffffffffff [commit: 172, lastindex: 174, lastterm: 6] starts to restore snapshot [index: 175, term: 6]
I161110 14:11:15.930422 1325 raft/log.go:288 [s3,r1/30:/M{in-ax}] log [committed=172, applied=172, unstable.offset=175, len(unstable.Entries)=0] starts to restore snapshot [index: 175, term: 6]
I161110 14:11:15.930438 1325 raft/raft.go:1135 [s3,r1/30:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 176, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.930451 1325 raft/raft.go:1135 [s3,r1/30:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 176, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:15.930462 1325 raft/raft.go:1102 [s3,r1/30:/M{in-ax}] ffffffffffffffff [commit: 175] restored snapshot [index: 175, term: 6]
I161110 14:11:15.930476 1325 storage/replica_raftstorage.go:589 [s3,r1/30:/M{in-ax}] applying Raft snapshot at index 175 (id=3ee80bbe, encoded size=16, 1 rocksdb batches, 45 log entries)
I161110 14:11:15.934541 1325 storage/replica_raftstorage.go:592 [s3,r1/30:/M{in-ax}] applied Raft snapshot in 0.004s
I161110 14:11:15.935080 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:31
I161110 14:11:15.935897 79 raft/raft.go:568 [s3,r1/30:/M{in-ax}] 1e became follower at term 6
I161110 14:11:15.936007 79 raft/raft.go:316 [s3,r1/30:/M{in-ax}] newRaft 1e [peers: [1,2], term: 6, commit: 175, applied: 175, lastindex: 175, lastterm: 6]
I161110 14:11:15.936801 1291 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:31}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:31}]
I161110 14:11:15.937372 274 storage/store.go:2984 [s3,r1/30:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:15.939457 201 storage/store.go:2037 [s3,r1/30:/M{in-ax}] DEL 1
W161110 14:11:15.946003 266 storage/store.go:2988 [s1] got error from range 1, replica {3 3 30}: raft group deleted
W161110 14:11:15.946025 266 storage/store.go:2988 [s1] got error from range 1, replica {3 3 30}: raft group deleted
W161110 14:11:15.983187 266 storage/store.go:2988 [s1] got error from range 1, replica {3 3 30}: raft group deleted
I161110 14:11:16.059421 75 raft/raft.go:568 [s3,r1/31:{-}] 1f became follower at term 0
I161110 14:11:16.059998 75 raft/raft.go:316 [s3,r1/31:{-}] newRaft 1f [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.060016 75 raft/raft.go:568 [s3,r1/31:{-}] 1f became follower at term 1
I161110 14:11:16.060066 75 raft/raft.go:710 [s3,r1/31:{-}] 1f [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:16.060092 75 raft/raft.go:568 [s3,r1/31:{-}] 1f became follower at term 6
I161110 14:11:16.060108 75 raft/raft.go:1088 [s3,r1/31:{-}] 1f [logterm: 0, index: 177] rejected msgApp [logterm: 6, index: 177] from 1
I161110 14:11:16.060400 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 1f for index 177
I161110 14:11:16.060556 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 1f to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.064138 54 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 4e85c20f at index 177 in 58.582µs.
I161110 14:11:16.071541 54 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 131, commit: 177] sent snapshot[index: 177, term: 6] to 1f [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.076809 54 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 1f [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 177]
I161110 14:11:16.085018 1329 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 253, log entries: 47
I161110 14:11:16.095239 1365 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.096080 1365 raft/raft.go:1125 [s3,r1/31:{-}] 1f [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 177, term: 6]
I161110 14:11:16.096112 1365 raft/log.go:288 [s3,r1/31:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 177, term: 6]
I161110 14:11:16.096140 1365 raft/raft.go:1135 [s3,r1/31:{-}] 1f restored progress of 1 [next = 178, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.096166 1365 raft/raft.go:1135 [s3,r1/31:{-}] 1f restored progress of 2 [next = 178, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.096211 1365 raft/raft.go:1135 [s3,r1/31:{-}] 1f restored progress of 1f [next = 178, match = 177, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.096229 1365 raft/raft.go:1102 [s3,r1/31:{-}] 1f [commit: 177] restored snapshot [index: 177, term: 6]
I161110 14:11:16.096338 1365 storage/replica_raftstorage.go:589 [s3,r1/31:{-}] applying Raft snapshot at index 177 (id=4e85c20f, encoded size=16, 1 rocksdb batches, 47 log entries)
I161110 14:11:16.104340 1365 storage/replica_raftstorage.go:592 [s3,r1/31:/M{in-ax}] applied Raft snapshot in 0.008s
I161110 14:11:16.104664 59 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 1f [next = 178, match = 177, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 177]
I161110 14:11:16.132067 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:31 > next_replica_id:32
I161110 14:11:16.133801 1382 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:31}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.147502 274 storage/store.go:2984 [s3,r1/31:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.150518 201 storage/store.go:2037 [s3,r1/31:/M{in-ax}] DEL 1
I161110 14:11:16.151236 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot df0a7f7b at index 182 in 51.835µs.
I161110 14:11:16.152216 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 259, log entries: 52
I161110 14:11:16.152511 1389 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.152567 1389 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.153196 1389 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.153244 1389 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.153355 1389 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.153380 1389 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:16.153989 1389 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 182, term: 6]
I161110 14:11:16.154008 1389 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 182, term: 6]
I161110 14:11:16.154026 1389 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 183, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.154039 1389 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 183, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.154049 1389 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 182] restored snapshot [index: 182, term: 6]
I161110 14:11:16.154065 1389 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 182 (id=df0a7f7b, encoded size=16, 1 rocksdb batches, 52 log entries)
I161110 14:11:16.173635 1389 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.020s
I161110 14:11:16.173683 1389 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:16.174411 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:32
I161110 14:11:16.176096 1336 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:32}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:32}]
I161110 14:11:16.177678 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:32 > next_replica_id:33
I161110 14:11:16.178205 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 20 became follower at term 6
I161110 14:11:16.178247 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 20 [peers: [1,2], term: 6, commit: 182, applied: 182, lastindex: 182, lastterm: 6]
I161110 14:11:16.178295 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 20 [logterm: 0, index: 184] rejected msgApp [logterm: 6, index: 184] from 1
I161110 14:11:16.178714 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 182) from 20 for index 184
I161110 14:11:16.178739 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 20 to [next = 183, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.179156 1410 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:32}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.180491 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 33763369 at index 186 in 40.338µs.
I161110 14:11:16.180821 274 storage/store.go:2984 [s3,r1/32:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.182040 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 265, log entries: 56
I161110 14:11:16.186608 201 storage/store.go:2037 [s3,r1/32:/M{in-ax}] DEL 1
I161110 14:11:16.186813 1398 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.187118 1398 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.187778 1398 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.187918 1398 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.187973 1398 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.187994 1398 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:16.189006 1398 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 186, term: 6]
I161110 14:11:16.189025 1398 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 186, term: 6]
I161110 14:11:16.189054 1398 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 187, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.189186 1398 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 187, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.189209 1398 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 186] restored snapshot [index: 186, term: 6]
I161110 14:11:16.189586 1398 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 186 (id=33763369, encoded size=16, 1 rocksdb batches, 56 log entries)
I161110 14:11:16.230289 1398 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.041s
I161110 14:11:16.230318 1398 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:16.231450 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:33
I161110 14:11:16.232850 1426 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:33}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:33}]
I161110 14:11:16.236390 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:33 > next_replica_id:34
I161110 14:11:16.240054 1442 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:33}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.240361 76 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 21 became follower at term 6
I161110 14:11:16.240437 76 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 21 [peers: [1,2], term: 6, commit: 186, applied: 186, lastindex: 186, lastterm: 6]
I161110 14:11:16.240466 76 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 21 [logterm: 0, index: 191] rejected msgApp [logterm: 6, index: 191] from 1
I161110 14:11:16.240604 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 186) from 21 for index 191
I161110 14:11:16.240662 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 21 to [next = 187, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.242435 274 storage/store.go:2984 [s3,r1/33:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.242660 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot d7ac6bab at index 193 in 41.038µs.
I161110 14:11:16.246314 201 storage/store.go:2037 [s3,r1/33:/M{in-ax}] DEL 1
I161110 14:11:16.249649 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 274, log entries: 63
I161110 14:11:16.250851 1475 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.250920 1475 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.252452 1475 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.252562 1475 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.252651 1475 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.252768 1475 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:16.254564 1475 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 193, term: 6]
I161110 14:11:16.254616 1475 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 193, term: 6]
I161110 14:11:16.254924 1475 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.255015 1475 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.255032 1475 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 193] restored snapshot [index: 193, term: 6]
I161110 14:11:16.255061 1475 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 193 (id=d7ac6bab, encoded size=16, 1 rocksdb batches, 63 log entries)
I161110 14:11:16.260327 1475 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161110 14:11:16.260921 1475 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:16.263705 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:34
I161110 14:11:16.267570 1463 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:34}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:34}]
I161110 14:11:16.270552 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:34 > next_replica_id:35
I161110 14:11:16.271394 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 22 became follower at term 6
I161110 14:11:16.271426 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 22 [peers: [1,2], term: 6, commit: 193, applied: 193, lastindex: 193, lastterm: 6]
I161110 14:11:16.271466 75 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 22 [logterm: 0, index: 195] rejected msgApp [logterm: 6, index: 195] from 1
I161110 14:11:16.271583 59 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 193) from 22 for index 195
I161110 14:11:16.271605 59 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 22 to [next = 194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.273077 1491 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:34}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.274427 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 9a76ed86 at index 197 in 36.002µs.
I161110 14:11:16.277771 201 storage/store.go:2037 [s3,r1/34:/M{in-ax}] DEL 1
I161110 14:11:16.278673 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 280, log entries: 67
I161110 14:11:16.278972 1503 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.279914 1503 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.301267 1503 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.301515 1503 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.301555 1503 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.302224 1503 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:16.303578 1503 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 197, term: 6]
I161110 14:11:16.303604 1503 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 197, term: 6]
I161110 14:11:16.303633 1503 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 198, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.303705 1503 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 198, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.303794 1503 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 197] restored snapshot [index: 197, term: 6]
I161110 14:11:16.303881 1503 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 197 (id=9a76ed86, encoded size=16, 1 rocksdb batches, 67 log entries)
I161110 14:11:16.476219 1503 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.172s
I161110 14:11:16.508286 1503 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:16.537890 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:35
I161110 14:11:16.540696 1505 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:35}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:35}]
I161110 14:11:16.558245 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 23 became follower at term 6
I161110 14:11:16.558278 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 23 [peers: [1,2], term: 6, commit: 197, applied: 197, lastindex: 197, lastterm: 6]
I161110 14:11:16.558317 77 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 23 [logterm: 0, index: 202] rejected msgApp [logterm: 6, index: 202] from 1
I161110 14:11:16.558651 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 197) from 23 for index 202
I161110 14:11:16.558678 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 23 to [next = 198, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.559743 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:35 > next_replica_id:36
I161110 14:11:16.562820 1519 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:35}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.564286 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 903051dc at index 207 in 39.833µs.
I161110 14:11:16.565176 274 storage/store.go:2984 [s3,r1/35:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.565519 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 292, log entries: 77
I161110 14:11:16.565886 1529 raft/raft.go:568 [s3,r1/35:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:16.565920 1529 raft/raft.go:316 [s3,r1/35:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 207, applied: 207, lastindex: 207, lastterm: 6]
I161110 14:11:16.565962 1529 raft/raft.go:1106 [s3,r1/35:/M{in-ax}] ffffffffffffffff [commit: 207] ignored snapshot [index: 207, term: 6]
I161110 14:11:16.568388 201 storage/store.go:2037 [s3,r1/35:/M{in-ax}] DEL 1
I161110 14:11:16.580855 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:36
I161110 14:11:16.588030 1436 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:36}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:36}]
I161110 14:11:16.906039 77 raft/raft.go:568 [s3,r1/36:{-}] 24 became follower at term 0
I161110 14:11:16.906855 77 raft/raft.go:316 [s3,r1/36:{-}] newRaft 24 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.906874 77 raft/raft.go:568 [s3,r1/36:{-}] 24 became follower at term 1
I161110 14:11:16.906894 77 raft/raft.go:710 [s3,r1/36:{-}] 24 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161110 14:11:16.906904 77 raft/raft.go:568 [s3,r1/36:{-}] 24 became follower at term 6
I161110 14:11:16.907030 77 raft/raft.go:1088 [s3,r1/36:{-}] 24 [logterm: 0, index: 209] rejected msgApp [logterm: 6, index: 209] from 1
I161110 14:11:16.907529 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 24 for index 209
I161110 14:11:16.907553 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 24 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.907635 58 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 1fe7b320 at index 210 in 46.626µs.
I161110 14:11:16.907665 58 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 131, commit: 212] sent snapshot[index: 210, term: 6] to 24 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.907681 58 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 24 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 210]
I161110 14:11:16.909290 1570 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 296, log entries: 80
I161110 14:11:16.909520 1557 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.910272 1557 raft/raft.go:1125 [s3,r1/36:{-}] 24 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 210, term: 6]
I161110 14:11:16.910289 1557 raft/log.go:288 [s3,r1/36:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 210, term: 6]
I161110 14:11:16.910304 1557 raft/raft.go:1135 [s3,r1/36:{-}] 24 restored progress of 1 [next = 211, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.910317 1557 raft/raft.go:1135 [s3,r1/36:{-}] 24 restored progress of 2 [next = 211, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.910331 1557 raft/raft.go:1135 [s3,r1/36:{-}] 24 restored progress of 24 [next = 211, match = 210, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.910343 1557 raft/raft.go:1102 [s3,r1/36:{-}] 24 [commit: 210] restored snapshot [index: 210, term: 6]
I161110 14:11:16.910364 1557 storage/replica_raftstorage.go:589 [s3,r1/36:{-}] applying Raft snapshot at index 210 (id=1fe7b320, encoded size=16, 1 rocksdb batches, 80 log entries)
I161110 14:11:16.913885 1557 storage/replica_raftstorage.go:592 [s3,r1/36:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:16.914136 56 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 24 [next = 211, match = 210, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 210]
I161110 14:11:16.973382 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:36 > next_replica_id:37
I161110 14:11:16.974925 1576 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:36}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.976146 274 storage/store.go:2984 [s3,r1/36:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.976752 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 6216b8b5 at index 217 in 39.028µs.
I161110 14:11:16.978048 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 302, log entries: 87
I161110 14:11:16.978872 201 storage/store.go:2037 [s3,r1/36:/M{in-ax}] DEL 1
I161110 14:11:16.979087 1582 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.979141 1582 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.979909 1582 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.979928 1582 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.979947 1582 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.979957 1582 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:16.980637 1582 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 217, term: 6]
I161110 14:11:16.980655 1582 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 217, term: 6]
I161110 14:11:16.980674 1582 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 218, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.980707 1582 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 218, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.980720 1582 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 217] restored snapshot [index: 217, term: 6]
I161110 14:11:16.980737 1582 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 217 (id=6216b8b5, encoded size=16, 1 rocksdb batches, 87 log entries)
I161110 14:11:16.984258 1582 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161110 14:11:16.984277 1582 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:16.984870 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:37
I161110 14:11:16.986085 1618 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:37}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:37}]
I161110 14:11:16.988040 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:37 > next_replica_id:38
I161110 14:11:16.988616 79 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 25 became follower at term 6
I161110 14:11:16.988650 79 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 25 [peers: [1,2], term: 6, commit: 217, applied: 217, lastindex: 217, lastterm: 6]
I161110 14:11:16.988667 79 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 25 [logterm: 0, index: 219] rejected msgApp [logterm: 6, index: 219] from 1
I161110 14:11:16.988823 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 217) from 25 for index 219
I161110 14:11:16.988848 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 25 to [next = 218, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:16.989458 1591 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:37}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:16.991055 274 storage/store.go:2984 [s3,r1/37:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:16.991141 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 42eb678d at index 221 in 42.371µs.
I161110 14:11:16.995968 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 308, log entries: 91
I161110 14:11:16.996964 201 storage/store.go:2037 [s3,r1/37:/M{in-ax}] DEL 1
I161110 14:11:16.997182 1636 storage/store.go:1908 [s3] ADD 1
I161110 14:11:16.997362 1636 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:16.999421 1636 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:16.999444 1636 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:16.999467 1636 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:16.999479 1636 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:17.000195 1636 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 221, term: 6]
I161110 14:11:17.000215 1636 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 221, term: 6]
I161110 14:11:17.000234 1636 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 222, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.000247 1636 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 222, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.000257 1636 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 221] restored snapshot [index: 221, term: 6]
I161110 14:11:17.000276 1636 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 221 (id=42eb678d, encoded size=16, 1 rocksdb batches, 91 log entries)
I161110 14:11:17.004641 1636 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161110 14:11:17.004662 1636 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.005185 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:38
I161110 14:11:17.006852 1638 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:38}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:38}]
I161110 14:11:17.008622 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:38 > next_replica_id:39
I161110 14:11:17.009082 81 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 26 became follower at term 6
I161110 14:11:17.009177 81 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 26 [peers: [1,2], term: 6, commit: 221, applied: 221, lastindex: 221, lastterm: 6]
I161110 14:11:17.009195 81 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 26 [logterm: 0, index: 223] rejected msgApp [logterm: 6, index: 223] from 1
I161110 14:11:17.009327 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 221) from 26 for index 223
I161110 14:11:17.009347 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 26 to [next = 222, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.010296 1609 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:38}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.011565 274 storage/store.go:2984 [s3,r1/38:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:17.012144 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 2f439764 at index 225 in 41.954µs.
I161110 14:11:17.013363 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 314, log entries: 95
I161110 14:11:17.014741 201 storage/store.go:2037 [s3,r1/38:/M{in-ax}] DEL 1
I161110 14:11:17.014942 1601 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.014989 1601 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:17.015720 1601 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.015736 1601 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:17.015756 1601 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:17.015767 1601 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:17.016413 1601 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 225, term: 6]
I161110 14:11:17.016430 1601 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 225, term: 6]
I161110 14:11:17.016451 1601 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 226, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.016465 1601 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 226, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.016476 1601 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 225] restored snapshot [index: 225, term: 6]
I161110 14:11:17.016493 1601 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 225 (id=2f439764, encoded size=16, 1 rocksdb batches, 95 log entries)
I161110 14:11:17.020296 1601 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161110 14:11:17.020322 1601 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.020841 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:39
I161110 14:11:17.022333 1630 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:39}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:39}]
I161110 14:11:17.024035 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:39 > next_replica_id:40
I161110 14:11:17.024488 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 27 became follower at term 6
I161110 14:11:17.024519 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 27 [peers: [1,2], term: 6, commit: 225, applied: 225, lastindex: 225, lastterm: 6]
I161110 14:11:17.024537 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 27 [logterm: 0, index: 227] rejected msgApp [logterm: 6, index: 227] from 1
I161110 14:11:17.024896 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 225) from 27 for index 227
I161110 14:11:17.024935 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 27 to [next = 226, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.025344 1657 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:39}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.027784 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 32409aec at index 229 in 48.1µs.
I161110 14:11:17.028203 274 storage/store.go:2984 [s3,r1/39:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:17.029005 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 320, log entries: 99
I161110 14:11:17.031021 201 storage/store.go:2037 [s3,r1/39:/M{in-ax}] DEL 1
I161110 14:11:17.031239 1661 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.031290 1661 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:17.032097 1661 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.053689 1661 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:17.063944 1661 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:17.072461 1661 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:17.125263 1661 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 229, term: 6]
I161110 14:11:17.125304 1661 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 229, term: 6]
I161110 14:11:17.125365 1661 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 230, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.125428 1661 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 230, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.125463 1661 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 229] restored snapshot [index: 229, term: 6]
I161110 14:11:17.125497 1661 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 229 (id=32409aec, encoded size=16, 1 rocksdb batches, 99 log entries)
I161110 14:11:17.175240 1661 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.050s
I161110 14:11:17.175271 1661 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.182005 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:40
I161110 14:11:17.183156 1715 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:40}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:40}]
I161110 14:11:17.187272 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:40 > next_replica_id:41
I161110 14:11:17.207540 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 28 became follower at term 6
I161110 14:11:17.207591 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 28 [peers: [1,2], term: 6, commit: 229, applied: 229, lastindex: 229, lastterm: 6]
I161110 14:11:17.207619 77 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 28 [logterm: 0, index: 234] rejected msgApp [logterm: 6, index: 234] from 1
I161110 14:11:17.207798 59 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 229) from 28 for index 234
I161110 14:11:17.207825 59 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 28 to [next = 230, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.214084 1676 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:40}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.214410 91 storage/raft_log_queue.go:219 [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 131-235
I161110 14:11:17.219449 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot a2c4ae81 at index 236 in 42.22µs.
I161110 14:11:17.220755 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 329, log entries: 106
I161110 14:11:17.225170 201 storage/store.go:2037 [s3,r1/40:/M{in-ax}] DEL 1
I161110 14:11:17.225516 1733 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.225563 1733 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:17.226368 1733 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.226383 1733 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:17.226403 1733 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:17.226414 1733 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:17.236821 1733 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 236, term: 6]
I161110 14:11:17.237089 1733 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 236, term: 6]
I161110 14:11:17.237136 1733 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.237161 1733 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.237179 1733 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 236] restored snapshot [index: 236, term: 6]
I161110 14:11:17.237200 1733 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 236 (id=a2c4ae81, encoded size=16, 1 rocksdb batches, 106 log entries)
I161110 14:11:17.243401 1733 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.006s
I161110 14:11:17.243424 1733 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.244022 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:41
I161110 14:11:17.245371 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:17.248313 201 storage/store.go:2037 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.249937 1738 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:41}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:41}]
I161110 14:11:17.276517 79 raft/raft.go:568 [s3,r1/41:{-}] 29 became follower at term 0
I161110 14:11:17.277361 79 raft/raft.go:316 [s3,r1/41:{-}] newRaft 29 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.277379 79 raft/raft.go:568 [s3,r1/41:{-}] 29 became follower at term 1
I161110 14:11:17.277399 79 raft/raft.go:710 [s3,r1/41:{-}] 29 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:17.277409 79 raft/raft.go:568 [s3,r1/41:{-}] 29 became follower at term 6
I161110 14:11:17.277425 79 raft/raft.go:1088 [s3,r1/41:{-}] 29 [logterm: 0, index: 239] rejected msgApp [logterm: 6, index: 239] from 1
I161110 14:11:17.279081 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 29 for index 239
I161110 14:11:17.280153 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 29 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.281416 58 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 2e45d377 at index 240 in 1.20172ms.
I161110 14:11:17.281456 58 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 235, commit: 240] sent snapshot[index: 240, term: 6] to 29 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.281479 58 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 29 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 240]
I161110 14:11:17.326041 1679 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 333, log entries: 6
I161110 14:11:17.326164 1749 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.531058 1749 raft/raft.go:1125 [s3,r1/41:{-}] 29 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 240, term: 6]
I161110 14:11:17.531404 1749 raft/log.go:288 [s3,r1/41:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 240, term: 6]
I161110 14:11:17.531427 1749 raft/raft.go:1135 [s3,r1/41:{-}] 29 restored progress of 1 [next = 241, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.531443 1749 raft/raft.go:1135 [s3,r1/41:{-}] 29 restored progress of 2 [next = 241, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.531456 1749 raft/raft.go:1135 [s3,r1/41:{-}] 29 restored progress of 29 [next = 241, match = 240, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.531468 1749 raft/raft.go:1102 [s3,r1/41:{-}] 29 [commit: 240] restored snapshot [index: 240, term: 6]
I161110 14:11:17.531490 1749 storage/replica_raftstorage.go:589 [s3,r1/41:{-}] applying Raft snapshot at index 240 (id=2e45d377, encoded size=16, 1 rocksdb batches, 6 log entries)
I161110 14:11:17.534265 1749 storage/replica_raftstorage.go:592 [s3,r1/41:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:17.534988 52 raft/raft.go:878 [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 29 [next = 241, match = 240, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 240]
I161110 14:11:17.650079 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:41 > next_replica_id:42
I161110 14:11:17.653631 1784 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:41}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.654804 274 storage/store.go:2984 [s3,r1/41:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:17.655631 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 0153692d at index 250 in 68.667µs.
I161110 14:11:17.662040 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 344, log entries: 16
I161110 14:11:17.662591 201 storage/store.go:2037 [s3,r1/41:/M{in-ax}] DEL 1
I161110 14:11:17.662836 1772 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.662886 1772 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:17.663931 1772 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.663952 1772 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:17.663970 1772 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:17.663979 1772 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:17.666669 1772 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 250, term: 6]
I161110 14:11:17.666819 1772 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 250, term: 6]
I161110 14:11:17.666863 1772 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 251, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.666908 1772 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 251, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.666943 1772 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 250] restored snapshot [index: 250, term: 6]
I161110 14:11:17.666962 1772 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 250 (id=0153692d, encoded size=16, 1 rocksdb batches, 16 log entries)
I161110 14:11:17.672485 1772 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161110 14:11:17.672513 1772 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:17.672916 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 29 became follower at term 6
I161110 14:11:17.672950 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 29 [peers: [1,2], term: 6, commit: 250, applied: 250, lastindex: 250, lastterm: 6]
I161110 14:11:17.673392 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:42
I161110 14:11:17.675135 1809 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:42}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:42}]
I161110 14:11:17.676801 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:42 > next_replica_id:43
I161110 14:11:17.678279 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 2a became follower at term 6
I161110 14:11:17.678314 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 2a [peers: [1,2], term: 6, commit: 250, applied: 250, lastindex: 250, lastterm: 6]
I161110 14:11:17.678331 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 2a [logterm: 0, index: 252] rejected msgApp [logterm: 6, index: 252] from 1
I161110 14:11:17.679250 57 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 250) from 2a for index 252
I161110 14:11:17.679275 57 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2a to [next = 251, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.680094 1821 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:42}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.682625 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 7e918f80 at index 254 in 40.369µs.
I161110 14:11:17.685225 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 350, log entries: 20
I161110 14:11:17.685939 1832 raft/raft.go:568 [s3,r1/42:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:17.685977 1832 raft/raft.go:316 [s3,r1/42:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 254, applied: 254, lastindex: 254, lastterm: 6]
I161110 14:11:17.686025 1832 raft/raft.go:1106 [s3,r1/42:/M{in-ax}] ffffffffffffffff [commit: 254] ignored snapshot [index: 254, term: 6]
I161110 14:11:17.686605 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:43
I161110 14:11:17.690561 1791 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:43}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:43}]
I161110 14:11:17.703801 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:43 > next_replica_id:44
I161110 14:11:17.704297 75 raft/raft.go:568 [s3,r1/42:/M{in-ax}] 2b became follower at term 6
I161110 14:11:17.704333 75 raft/raft.go:316 [s3,r1/42:/M{in-ax}] newRaft 2b [peers: [1,2], term: 6, commit: 254, applied: 254, lastindex: 254, lastterm: 6]
I161110 14:11:17.704351 75 raft/raft.go:1088 [s3,r1/42:/M{in-ax}] 2b [logterm: 0, index: 256] rejected msgApp [logterm: 6, index: 256] from 1
I161110 14:11:17.704659 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 254) from 2b for index 256
I161110 14:11:17.704681 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2b to [next = 255, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.705097 1759 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:43}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.706539 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 5808b5df at index 258 in 41.198µs.
I161110 14:11:17.707544 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 356, log entries: 24
I161110 14:11:17.707716 1842 raft/raft.go:568 [s3,r1/43:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:17.707752 1842 raft/raft.go:316 [s3,r1/43:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,2b], term: 6, commit: 257, applied: 257, lastindex: 257, lastterm: 6]
I161110 14:11:17.707861 1842 raft/raft.go:1125 [s3,r1/43:/M{in-ax}] ffffffffffffffff [commit: 257, lastindex: 257, lastterm: 6] starts to restore snapshot [index: 258, term: 6]
I161110 14:11:17.707877 1842 raft/log.go:288 [s3,r1/43:/M{in-ax}] log [committed=257, applied=257, unstable.offset=258, len(unstable.Entries)=0] starts to restore snapshot [index: 258, term: 6]
I161110 14:11:17.707896 1842 raft/raft.go:1135 [s3,r1/43:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 259, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.707917 1842 raft/raft.go:1135 [s3,r1/43:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 259, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.707928 1842 raft/raft.go:1102 [s3,r1/43:/M{in-ax}] ffffffffffffffff [commit: 258] restored snapshot [index: 258, term: 6]
I161110 14:11:17.707943 1842 storage/replica_raftstorage.go:589 [s3,r1/43:/M{in-ax}] applying Raft snapshot at index 258 (id=5808b5df, encoded size=16, 1 rocksdb batches, 24 log entries)
I161110 14:11:17.713827 1842 storage/replica_raftstorage.go:592 [s3,r1/43:/M{in-ax}] applied Raft snapshot in 0.006s
I161110 14:11:17.716295 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:44
I161110 14:11:17.717413 1841 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:44}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:44}]
I161110 14:11:17.719124 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:44 > next_replica_id:45
I161110 14:11:17.720195 79 raft/raft.go:568 [s3,r1/43:/M{in-ax}] 2c became follower at term 6
I161110 14:11:17.720265 79 raft/raft.go:316 [s3,r1/43:/M{in-ax}] newRaft 2c [peers: [1,2], term: 6, commit: 258, applied: 258, lastindex: 258, lastterm: 6]
I161110 14:11:17.720416 79 raft/raft.go:1088 [s3,r1/43:/M{in-ax}] 2c [logterm: 0, index: 260] rejected msgApp [logterm: 6, index: 260] from 1
I161110 14:11:17.720968 56 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 258) from 2c for index 260
I161110 14:11:17.721005 56 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2c to [next = 259, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.721777 1875 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:44}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.728283 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot c57e1413 at index 262 in 43.41µs.
I161110 14:11:17.729926 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 362, log entries: 28
I161110 14:11:17.730304 1895 raft/raft.go:568 [s3,r1/44:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:17.730366 1895 raft/raft.go:316 [s3,r1/44:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,2c], term: 6, commit: 261, applied: 261, lastindex: 261, lastterm: 6]
I161110 14:11:17.730443 1895 raft/raft.go:1125 [s3,r1/44:/M{in-ax}] ffffffffffffffff [commit: 261, lastindex: 261, lastterm: 6] starts to restore snapshot [index: 262, term: 6]
I161110 14:11:17.730472 1895 raft/log.go:288 [s3,r1/44:/M{in-ax}] log [committed=261, applied=261, unstable.offset=262, len(unstable.Entries)=0] starts to restore snapshot [index: 262, term: 6]
I161110 14:11:17.730538 1895 raft/raft.go:1135 [s3,r1/44:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 263, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.730567 1895 raft/raft.go:1135 [s3,r1/44:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 263, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.730589 1895 raft/raft.go:1102 [s3,r1/44:/M{in-ax}] ffffffffffffffff [commit: 262] restored snapshot [index: 262, term: 6]
I161110 14:11:17.730614 1895 storage/replica_raftstorage.go:589 [s3,r1/44:/M{in-ax}] applying Raft snapshot at index 262 (id=c57e1413, encoded size=16, 1 rocksdb batches, 28 log entries)
I161110 14:11:17.767291 1895 storage/replica_raftstorage.go:592 [s3,r1/44:/M{in-ax}] applied Raft snapshot in 0.037s
I161110 14:11:17.770309 201 storage/store.go:2037 [s3,r1/44:/M{in-ax}] DEL 1
I161110 14:11:17.770407 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:45
I161110 14:11:17.779988 1857 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:45}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:45}]
I161110 14:11:17.806427 81 raft/raft.go:568 [s3,r1/45:{-}] 2d became follower at term 0
I161110 14:11:17.807365 81 raft/raft.go:316 [s3,r1/45:{-}] newRaft 2d [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:17.807387 81 raft/raft.go:568 [s3,r1/45:{-}] 2d became follower at term 1
I161110 14:11:17.807410 81 raft/raft.go:710 [s3,r1/45:{-}] 2d [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:17.807422 81 raft/raft.go:568 [s3,r1/45:{-}] 2d became follower at term 6
I161110 14:11:17.807442 81 raft/raft.go:1088 [s3,r1/45:{-}] 2d [logterm: 0, index: 264] rejected msgApp [logterm: 6, index: 264] from 1
I161110 14:11:17.808513 53 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 2d for index 264
I161110 14:11:17.808540 53 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2d to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.808633 53 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 6810ce0f at index 265 in 55.672µs.
I161110 14:11:17.808662 53 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 235, commit: 265] sent snapshot[index: 265, term: 6] to 2d [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.808676 53 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 2d [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 265]
I161110 14:11:17.810581 1928 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 366, log entries: 31
I161110 14:11:17.810712 1931 storage/store.go:1908 [s3] ADD 1
I161110 14:11:17.811687 1931 raft/raft.go:1125 [s3,r1/45:{-}] 2d [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 265, term: 6]
I161110 14:11:17.811710 1931 raft/log.go:288 [s3,r1/45:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 265, term: 6]
I161110 14:11:17.811731 1931 raft/raft.go:1135 [s3,r1/45:{-}] 2d restored progress of 1 [next = 266, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.811748 1931 raft/raft.go:1135 [s3,r1/45:{-}] 2d restored progress of 2 [next = 266, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.811763 1931 raft/raft.go:1135 [s3,r1/45:{-}] 2d restored progress of 2d [next = 266, match = 265, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.811774 1931 raft/raft.go:1102 [s3,r1/45:{-}] 2d [commit: 265] restored snapshot [index: 265, term: 6]
I161110 14:11:17.811798 1931 storage/replica_raftstorage.go:589 [s3,r1/45:{-}] applying Raft snapshot at index 265 (id=6810ce0f, encoded size=16, 1 rocksdb batches, 31 log entries)
I161110 14:11:17.857328 1931 storage/replica_raftstorage.go:592 [s3,r1/45:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:17.910594 1928 raft/raft.go:933 [s1,r1/1:/M{in-ax}] 1 snapshot succeeded, resumed sending replication messages to 2d [next = 266, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:17.931890 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:45 > next_replica_id:46
I161110 14:11:17.934094 1920 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:45}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.937509 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot a6bc16cc at index 269 in 40.547µs.
I161110 14:11:17.958693 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 371, log entries: 35
I161110 14:11:17.961639 1947 raft/raft.go:568 [s3,r1/45:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:17.961681 1947 raft/raft.go:316 [s3,r1/45:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 269, applied: 269, lastindex: 269, lastterm: 6]
I161110 14:11:17.961732 1947 raft/raft.go:1106 [s3,r1/45:/M{in-ax}] ffffffffffffffff [commit: 269] ignored snapshot [index: 269, term: 6]
I161110 14:11:17.962019 274 storage/store.go:2984 [s3,r1/45:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:17.962520 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:46
I161110 14:11:17.964783 1951 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:46}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:46}]
I161110 14:11:17.967496 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:46 > next_replica_id:47
I161110 14:11:17.969720 1963 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:46}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:17.971174 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 5a615568 at index 273 in 41.956µs.
I161110 14:11:18.023556 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 377, log entries: 39
I161110 14:11:18.096652 201 storage/store.go:2037 [s3,r1/45:/M{in-ax}] DEL 1
I161110 14:11:18.097266 79 raft/raft.go:568 [s3,r1/46:{-}] 2e became follower at term 0
I161110 14:11:18.098204 79 raft/raft.go:316 [s3,r1/46:{-}] newRaft 2e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.098228 79 raft/raft.go:568 [s3,r1/46:{-}] 2e became follower at term 1
I161110 14:11:18.098256 79 raft/raft.go:710 [s3,r1/46:{-}] 2e [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:18.098275 79 raft/raft.go:568 [s3,r1/46:{-}] 2e became follower at term 6
I161110 14:11:18.098296 79 raft/raft.go:1088 [s3,r1/46:{-}] 2e [logterm: 0, index: 271] rejected msgApp [logterm: 6, index: 271] from 1
E161110 14:11:18.099871 274 storage/store.go:2982 [s3,r1/46:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:18.122964 1974 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.123050 1974 raft/raft.go:568 [s3,r1/46:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:18.124176 1974 raft/raft.go:316 [s3,r1/46:{-}] newRaft ffffffffffffffff [peers: [], term: 6, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.124196 1974 raft/raft.go:568 [s3,r1/46:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:18.124229 1974 raft/raft.go:710 [s3,r1/46:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:18.124244 1974 raft/raft.go:568 [s3,r1/46:{-}] ffffffffffffffff became follower at term 6
E161110 14:11:18.124409 274 storage/store.go:2982 [s3,r1/46:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:18.125257 1974 raft/raft.go:1125 [s3,r1/46:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 273, term: 6]
I161110 14:11:18.125279 1974 raft/log.go:288 [s3,r1/46:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 273, term: 6]
I161110 14:11:18.125298 1974 raft/raft.go:1135 [s3,r1/46:{-}] ffffffffffffffff restored progress of 1 [next = 274, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.125311 1974 raft/raft.go:1135 [s3,r1/46:{-}] ffffffffffffffff restored progress of 2 [next = 274, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.125323 1974 raft/raft.go:1102 [s3,r1/46:{-}] ffffffffffffffff [commit: 273] restored snapshot [index: 273, term: 6]
I161110 14:11:18.125340 1974 storage/replica_raftstorage.go:589 [s3,r1/46:{-}] applying Raft snapshot at index 273 (id=5a615568, encoded size=16, 1 rocksdb batches, 39 log entries)
I161110 14:11:18.128804 1974 storage/replica_raftstorage.go:592 [s3,r1/46:/M{in-ax}] applied Raft snapshot in 0.003s
I161110 14:11:18.128826 1974 storage/store.go:2813 [s3,r1/46:/M{in-ax}] DEL 1
I161110 14:11:18.129327 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:47
I161110 14:11:18.133583 1882 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:47}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:47}]
I161110 14:11:18.139361 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:47 > next_replica_id:48
I161110 14:11:18.139869 75 raft/raft.go:568 [s3,r1/46:/M{in-ax}] 2f became follower at term 6
I161110 14:11:18.140001 75 raft/raft.go:316 [s3,r1/46:/M{in-ax}] newRaft 2f [peers: [1,2], term: 6, commit: 273, applied: 273, lastindex: 273, lastterm: 6]
I161110 14:11:18.140020 75 raft/raft.go:1088 [s3,r1/46:/M{in-ax}] 2f [logterm: 0, index: 278] rejected msgApp [logterm: 6, index: 278] from 1
I161110 14:11:18.140356 54 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 273) from 2f for index 278
I161110 14:11:18.140378 54 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 2f to [next = 274, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.141160 1868 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:47}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:18.142909 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 862af93c at index 280 in 36.135µs.
I161110 14:11:18.143375 274 storage/store.go:2984 [s3,r1/47:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:18.144160 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 386, log entries: 46
I161110 14:11:18.144375 1998 raft/raft.go:568 [s3,r1/47:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:18.144587 1998 raft/raft.go:316 [s3,r1/47:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,2f], term: 6, commit: 279, applied: 279, lastindex: 279, lastterm: 6]
I161110 14:11:18.144651 1998 raft/raft.go:1125 [s3,r1/47:/M{in-ax}] ffffffffffffffff [commit: 279, lastindex: 279, lastterm: 6] starts to restore snapshot [index: 280, term: 6]
I161110 14:11:18.144668 1998 raft/log.go:288 [s3,r1/47:/M{in-ax}] log [committed=279, applied=279, unstable.offset=280, len(unstable.Entries)=0] starts to restore snapshot [index: 280, term: 6]
I161110 14:11:18.144710 1998 raft/raft.go:1135 [s3,r1/47:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 281, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.144728 1998 raft/raft.go:1135 [s3,r1/47:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 281, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.144739 1998 raft/raft.go:1102 [s3,r1/47:/M{in-ax}] ffffffffffffffff [commit: 280] restored snapshot [index: 280, term: 6]
I161110 14:11:18.144756 1998 storage/replica_raftstorage.go:589 [s3,r1/47:/M{in-ax}] applying Raft snapshot at index 280 (id=862af93c, encoded size=16, 1 rocksdb batches, 46 log entries)
I161110 14:11:18.152139 1998 storage/replica_raftstorage.go:592 [s3,r1/47:/M{in-ax}] applied Raft snapshot in 0.007s
I161110 14:11:18.152764 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:48
I161110 14:11:18.154250 1872 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:48}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:48}]
I161110 14:11:18.155382 201 storage/store.go:2037 [s3,r1/47:/M{in-ax}] DEL 1
I161110 14:11:18.205147 79 raft/raft.go:568 [s3,r1/48:{-}] 30 became follower at term 0
I161110 14:11:18.293748 79 raft/raft.go:316 [s3,r1/48:{-}] newRaft 30 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.304719 79 raft/raft.go:568 [s3,r1/48:{-}] 30 became follower at term 1
I161110 14:11:18.336733 79 raft/raft.go:710 [s3,r1/48:{-}] 30 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:18.356889 79 raft/raft.go:568 [s3,r1/48:{-}] 30 became follower at term 6
I161110 14:11:18.356912 79 raft/raft.go:1088 [s3,r1/48:{-}] 30 [logterm: 0, index: 282] rejected msgApp [logterm: 6, index: 282] from 1
I161110 14:11:18.378613 79 raft/raft.go:1088 [s3,r1/48:{-}] 30 [logterm: 0, index: 282] rejected msgApp [logterm: 6, index: 282] from 1
I161110 14:11:18.382250 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 30 for index 282
I161110 14:11:18.382277 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 30 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.382411 52 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 2ba9edb9 at index 285 in 58.718µs.
I161110 14:11:18.382449 52 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 235, commit: 285] sent snapshot[index: 285, term: 6] to 30 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.382485 52 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 30 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 285]
I161110 14:11:18.382510 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 30 for index 282
I161110 14:11:18.383811 2011 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 392, log entries: 51
I161110 14:11:18.383945 2014 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.385232 2014 raft/raft.go:1125 [s3,r1/48:{-}] 30 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 285, term: 6]
I161110 14:11:18.385255 2014 raft/log.go:288 [s3,r1/48:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 285, term: 6]
I161110 14:11:18.385274 2014 raft/raft.go:1135 [s3,r1/48:{-}] 30 restored progress of 1 [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.385291 2014 raft/raft.go:1135 [s3,r1/48:{-}] 30 restored progress of 2 [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.385316 2014 raft/raft.go:1135 [s3,r1/48:{-}] 30 restored progress of 30 [next = 286, match = 285, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.385348 2014 raft/raft.go:1102 [s3,r1/48:{-}] 30 [commit: 285] restored snapshot [index: 285, term: 6]
I161110 14:11:18.385378 2014 storage/replica_raftstorage.go:589 [s3,r1/48:{-}] applying Raft snapshot at index 285 (id=2ba9edb9, encoded size=16, 1 rocksdb batches, 51 log entries)
I161110 14:11:18.399859 2014 storage/replica_raftstorage.go:592 [s3,r1/48:/M{in-ax}] applied Raft snapshot in 0.014s
I161110 14:11:18.400193 2011 raft/raft.go:933 [s1,r1/1:/M{in-ax}] 1 snapshot succeeded, resumed sending replication messages to 30 [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.401249 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:48 > next_replica_id:49
I161110 14:11:18.405211 2057 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:48}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:18.418267 201 storage/store.go:2037 [s3,r1/48:/M{in-ax}] DEL 1
I161110 14:11:18.433852 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 74b839db at index 290 in 237.843µs.
I161110 14:11:18.436115 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 398, log entries: 56
I161110 14:11:18.438877 2043 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.438992 2043 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:18.440137 2043 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.440157 2043 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:18.440178 2043 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:18.440189 2043 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:18.441229 2043 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 290, term: 6]
I161110 14:11:18.441253 2043 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 290, term: 6]
I161110 14:11:18.441273 2043 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.441311 2043 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.441327 2043 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 290] restored snapshot [index: 290, term: 6]
I161110 14:11:18.441344 2043 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 290 (id=74b839db, encoded size=16, 1 rocksdb batches, 56 log entries)
I161110 14:11:18.446318 2043 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161110 14:11:18.446343 2043 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:18.448117 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:49
I161110 14:11:18.450245 2071 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:49}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:49}]
I161110 14:11:18.453109 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:49 > next_replica_id:50
I161110 14:11:18.457726 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 31 became follower at term 6
I161110 14:11:18.457763 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 31 [peers: [1,2], term: 6, commit: 290, applied: 290, lastindex: 290, lastterm: 6]
I161110 14:11:18.457808 77 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 31 [logterm: 0, index: 292] rejected msgApp [logterm: 6, index: 292] from 1
I161110 14:11:18.457932 59 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 290) from 31 for index 292
I161110 14:11:18.457955 59 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 31 to [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.459453 2083 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:49}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:18.460847 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 6b453a6e at index 294 in 40.132µs.
I161110 14:11:18.463155 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 404, log entries: 60
I161110 14:11:18.464922 201 storage/store.go:2037 [s3,r1/49:/M{in-ax}] DEL 1
I161110 14:11:18.465147 2062 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.465201 2062 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:18.466220 2062 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.466236 2062 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:18.466258 2062 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:18.466271 2062 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:18.467243 2062 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 294, term: 6]
I161110 14:11:18.467264 2062 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 294, term: 6]
I161110 14:11:18.467283 2062 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 295, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.467299 2062 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 295, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.467311 2062 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 294] restored snapshot [index: 294, term: 6]
I161110 14:11:18.467328 2062 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 294 (id=6b453a6e, encoded size=16, 1 rocksdb batches, 60 log entries)
E161110 14:11:18.473474 274 storage/store.go:2982 [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:18.869692 2062 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.402s
I161110 14:11:18.869727 2062 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:18.870344 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:50
I161110 14:11:18.871757 2128 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:50}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:50}]
I161110 14:11:18.873532 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:50 > next_replica_id:51
I161110 14:11:18.874082 75 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 32 became follower at term 6
I161110 14:11:18.874161 75 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 32 [peers: [1,2], term: 6, commit: 294, applied: 294, lastindex: 294, lastterm: 6]
I161110 14:11:18.874201 75 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 32 [logterm: 0, index: 302] rejected msgApp [logterm: 6, index: 302] from 1
I161110 14:11:18.874362 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 294) from 32 for index 302
I161110 14:11:18.874383 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 32 to [next = 295, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.875212 2108 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:50}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:18.876936 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot eb5829d0 at index 304 in 42.639µs.
I161110 14:11:18.876986 274 storage/store.go:2984 [s3,r1/50:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:18.878093 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 416, log entries: 70
I161110 14:11:18.882335 201 storage/store.go:2037 [s3,r1/50:/M{in-ax}] DEL 1
I161110 14:11:18.882528 2164 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.882573 2164 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:18.883589 2164 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.883606 2164 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:18.883624 2164 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:18.883634 2164 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:18.884607 2164 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 304, term: 6]
I161110 14:11:18.884627 2164 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 304, term: 6]
I161110 14:11:18.884643 2164 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 305, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.884655 2164 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 305, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.884666 2164 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 304] restored snapshot [index: 304, term: 6]
I161110 14:11:18.884682 2164 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 304 (id=eb5829d0, encoded size=16, 1 rocksdb batches, 70 log entries)
I161110 14:11:18.889037 2164 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161110 14:11:18.889060 2164 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:18.889538 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:51
I161110 14:11:18.892169 2153 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:51}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:51}]
I161110 14:11:18.894515 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:51 > next_replica_id:52
I161110 14:11:18.895658 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 33 became follower at term 6
I161110 14:11:18.895693 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 33 [peers: [1,2], term: 6, commit: 304, applied: 304, lastindex: 304, lastterm: 6]
I161110 14:11:18.895709 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 33 [logterm: 0, index: 309] rejected msgApp [logterm: 6, index: 309] from 1
I161110 14:11:18.895827 57 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 304) from 33 for index 309
I161110 14:11:18.895847 57 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 33 to [next = 305, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.896604 2190 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:51}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:18.898351 274 storage/store.go:2984 [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:18.898396 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 3e686798 at index 311 in 59.938µs.
I161110 14:11:18.899774 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 425, log entries: 77
I161110 14:11:18.902489 201 storage/store.go:2037 [s3,r1/51:/M{in-ax}] DEL 1
I161110 14:11:18.902700 2195 storage/store.go:1908 [s3] ADD 1
I161110 14:11:18.902752 2195 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:18.904213 2195 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:18.904477 2195 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:18.904504 2195 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:18.904517 2195 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:18.905535 2195 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 311, term: 6]
I161110 14:11:18.905556 2195 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 311, term: 6]
I161110 14:11:18.905581 2195 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 312, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.922394 2195 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 312, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:18.922875 2195 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 311] restored snapshot [index: 311, term: 6]
I161110 14:11:18.933941 2195 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 311 (id=3e686798, encoded size=16, 1 rocksdb batches, 77 log entries)
I161110 14:11:19.048386 2195 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.114s
I161110 14:11:19.048517 2195 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.049198 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:52
I161110 14:11:19.054086 2213 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:52}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:52}]
I161110 14:11:19.056710 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:52 > next_replica_id:53
I161110 14:11:19.061455 77 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 34 became follower at term 6
I161110 14:11:19.061529 77 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 34 [peers: [1,2], term: 6, commit: 311, applied: 311, lastindex: 311, lastterm: 6]
I161110 14:11:19.061548 77 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 34 [logterm: 0, index: 313] rejected msgApp [logterm: 6, index: 313] from 1
I161110 14:11:19.061678 53 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 311) from 34 for index 313
I161110 14:11:19.061700 53 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 34 to [next = 312, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.062675 2135 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:52}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.063897 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot ba0bf00e at index 315 in 49.861µs.
I161110 14:11:19.066105 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 431, log entries: 81
I161110 14:11:19.073839 201 storage/store.go:2037 [s3,r1/52:/M{in-ax}] DEL 1
I161110 14:11:19.074153 2177 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.074203 2177 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:19.075324 2177 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.075366 2177 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.075393 2177 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.075410 2177 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.076636 2177 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 315, term: 6]
I161110 14:11:19.076655 2177 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 315, term: 6]
I161110 14:11:19.076675 2177 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 316, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.076688 2177 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 316, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.076698 2177 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 315] restored snapshot [index: 315, term: 6]
I161110 14:11:19.080634 2177 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 315 (id=ba0bf00e, encoded size=16, 1 rocksdb batches, 81 log entries)
E161110 14:11:19.092440 274 storage/store.go:2982 [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161110 14:11:19.094226 274 storage/store.go:2982 [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161110 14:11:19.095300 274 storage/store.go:2982 [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:19.097560 2177 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.015s
I161110 14:11:19.097579 2177 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.098492 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:53
I161110 14:11:19.100137 2236 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:53}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:53}]
I161110 14:11:19.101760 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:53 > next_replica_id:54
I161110 14:11:19.102558 80 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 35 became follower at term 6
I161110 14:11:19.102596 80 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 35 [peers: [1,2], term: 6, commit: 315, applied: 315, lastindex: 315, lastterm: 6]
I161110 14:11:19.102637 80 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 35 [logterm: 0, index: 320] rejected msgApp [logterm: 6, index: 320] from 1
I161110 14:11:19.103304 52 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 315) from 35 for index 320
I161110 14:11:19.103329 52 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 35 to [next = 316, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.105260 2254 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:53}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.106691 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot a18e48f5 at index 322 in 39.229µs.
I161110 14:11:19.125519 274 storage/store.go:2984 [s3,r1/53:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.126230 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 440, log entries: 88
I161110 14:11:19.172495 201 storage/store.go:2037 [s3,r1/53:/M{in-ax}] DEL 1
I161110 14:11:19.172763 2266 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.172811 2266 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:19.173902 2266 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.173931 2266 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.173952 2266 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.173964 2266 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.175016 2266 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 322, term: 6]
I161110 14:11:19.175038 2266 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 322, term: 6]
I161110 14:11:19.175058 2266 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.175071 2266 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.175082 2266 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 322] restored snapshot [index: 322, term: 6]
I161110 14:11:19.175098 2266 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 322 (id=a18e48f5, encoded size=16, 1 rocksdb batches, 88 log entries)
I161110 14:11:19.186686 2266 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.010s
I161110 14:11:19.186727 2266 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.187265 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:54
I161110 14:11:19.228573 2269 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:54}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:54}]
I161110 14:11:19.234683 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:54 > next_replica_id:55
I161110 14:11:19.235890 76 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 36 became follower at term 6
I161110 14:11:19.235951 76 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 36 [peers: [1,2], term: 6, commit: 322, applied: 322, lastindex: 322, lastterm: 6]
I161110 14:11:19.235977 76 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 36 [logterm: 0, index: 324] rejected msgApp [logterm: 6, index: 324] from 1
I161110 14:11:19.239788 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 322) from 36 for index 324
I161110 14:11:19.239908 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 36 to [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.246602 2287 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:54}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.248565 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 261f7d16 at index 328 in 41.429µs.
I161110 14:11:19.250500 274 storage/store.go:2984 [s3,r1/54:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.250823 274 storage/store.go:2984 [s3,r1/54:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.253777 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 448, log entries: 94
I161110 14:11:19.255328 201 storage/store.go:2037 [s3,r1/54:/M{in-ax}] DEL 1
I161110 14:11:19.255553 2305 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.255603 2305 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:19.256790 2305 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.256812 2305 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.256833 2305 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.256843 2305 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.258229 2305 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 328, term: 6]
I161110 14:11:19.258252 2305 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 328, term: 6]
I161110 14:11:19.258272 2305 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 329, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.258284 2305 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 329, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.258304 2305 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 328] restored snapshot [index: 328, term: 6]
I161110 14:11:19.258325 2305 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 328 (id=261f7d16, encoded size=16, 1 rocksdb batches, 94 log entries)
I161110 14:11:19.302819 2305 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.044s
I161110 14:11:19.303150 2305 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.304509 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:55
I161110 14:11:19.307533 2310 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:55}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:55}]
I161110 14:11:19.312447 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:55 > next_replica_id:56
I161110 14:11:19.313809 76 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 37 became follower at term 6
I161110 14:11:19.313846 76 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 37 [peers: [1,2], term: 6, commit: 328, applied: 328, lastindex: 328, lastterm: 6]
I161110 14:11:19.313891 76 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 37 [logterm: 0, index: 331] rejected msgApp [logterm: 6, index: 331] from 1
I161110 14:11:19.314130 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 328) from 37 for index 331
I161110 14:11:19.314520 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 37 to [next = 329, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.317316 2339 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:55}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.319604 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot d4577ce7 at index 333 in 41.419µs.
I161110 14:11:19.325507 201 storage/store.go:2037 [s3,r1/55:/M{in-ax}] DEL 1
I161110 14:11:19.330619 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 455, log entries: 99
I161110 14:11:19.330814 2318 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.330867 2318 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:19.332029 2318 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.332047 2318 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.332066 2318 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.332078 2318 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.333184 2318 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 333, term: 6]
I161110 14:11:19.333205 2318 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 333, term: 6]
I161110 14:11:19.333224 2318 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 334, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.333238 2318 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 334, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.333249 2318 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 333] restored snapshot [index: 333, term: 6]
I161110 14:11:19.333266 2318 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 333 (id=d4577ce7, encoded size=16, 1 rocksdb batches, 99 log entries)
I161110 14:11:19.338936 2318 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.006s
I161110 14:11:19.338958 2318 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.339590 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:56
I161110 14:11:19.343058 2346 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:56}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:56}]
I161110 14:11:19.355010 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:56 > next_replica_id:57
I161110 14:11:19.356305 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 38 became follower at term 6
I161110 14:11:19.356341 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 38 [peers: [1,2], term: 6, commit: 333, applied: 333, lastindex: 333, lastterm: 6]
I161110 14:11:19.356383 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 38 [logterm: 0, index: 335] rejected msgApp [logterm: 6, index: 335] from 1
I161110 14:11:19.356500 58 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 333) from 38 for index 335
I161110 14:11:19.356520 58 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 38 to [next = 334, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.357148 2362 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:56}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.359250 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 9e9e7170 at index 337 in 35.676µs.
I161110 14:11:19.375882 274 storage/store.go:2984 [s3,r1/56:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.376734 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 461, log entries: 103
I161110 14:11:19.379905 201 storage/store.go:2037 [s3,r1/56:/M{in-ax}] DEL 1
I161110 14:11:19.383587 2373 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.383641 2373 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161110 14:11:19.384829 2373 raft/raft.go:316 [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.384854 2373 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.386671 2373 raft/raft.go:710 [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.402750 2373 raft/raft.go:568 [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.532406 91 storage/raft_log_queue.go:219 [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 235-340
W161110 14:11:19.535342 266 storage/store.go:2988 [s1] got error from range 1, replica {3 3 55}: raft group deleted
I161110 14:11:19.535531 2373 raft/raft.go:1125 [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 337, term: 6]
I161110 14:11:19.539713 2373 raft/log.go:288 [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 337, term: 6]
I161110 14:11:19.550496 2373 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.550553 2373 raft/raft.go:1135 [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.550574 2373 raft/raft.go:1102 [s3,r1/?:{-}] ffffffffffffffff [commit: 337] restored snapshot [index: 337, term: 6]
I161110 14:11:19.550596 2373 storage/replica_raftstorage.go:589 [s3,r1/?:{-}] applying preemptive snapshot at index 337 (id=9e9e7170, encoded size=16, 1 rocksdb batches, 103 log entries)
I161110 14:11:19.568144 2373 storage/replica_raftstorage.go:592 [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.018s
I161110 14:11:19.568165 2373 storage/store.go:2813 [s3,r1/?:/M{in-ax}] DEL 1
I161110 14:11:19.568691 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:57
I161110 14:11:19.569817 2141 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:57}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:57}]
I161110 14:11:19.571403 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:57 > next_replica_id:58
I161110 14:11:19.571868 78 raft/raft.go:568 [s3,r1/?:/M{in-ax}] 39 became follower at term 6
I161110 14:11:19.571906 78 raft/raft.go:316 [s3,r1/?:/M{in-ax}] newRaft 39 [peers: [1,2], term: 6, commit: 337, applied: 337, lastindex: 337, lastterm: 6]
I161110 14:11:19.571954 78 raft/raft.go:1088 [s3,r1/?:/M{in-ax}] 39 [logterm: 0, index: 343] rejected msgApp [logterm: 6, index: 343] from 1
I161110 14:11:19.573118 55 raft/raft.go:863 [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 337) from 39 for index 343
I161110 14:11:19.573147 55 raft/raft.go:865 [s1,r1/1:/M{in-ax}] 1 decreased progress of 39 to [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.573235 55 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 2419a90f at index 344 in 48.364µs.
I161110 14:11:19.573268 55 raft/raft.go:400 [s1,r1/1:/M{in-ax}] 1 [firstindex: 340, commit: 344] sent snapshot[index: 344, term: 6] to 39 [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.573285 55 raft/raft.go:402 [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 39 [next = 338, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 344]
I161110 14:11:19.573784 2395 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:57}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.574481 2392 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 470, log entries: 5
I161110 14:11:19.574630 2403 raft/raft.go:1125 [s3,r1/?:/M{in-ax}] 39 [commit: 337, lastindex: 337, lastterm: 6] starts to restore snapshot [index: 344, term: 6]
I161110 14:11:19.574648 2403 raft/log.go:288 [s3,r1/?:/M{in-ax}] log [committed=337, applied=337, unstable.offset=338, len(unstable.Entries)=0] starts to restore snapshot [index: 344, term: 6]
I161110 14:11:19.574676 2403 raft/raft.go:1135 [s3,r1/?:/M{in-ax}] 39 restored progress of 1 [next = 345, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.574720 2403 raft/raft.go:1135 [s3,r1/?:/M{in-ax}] 39 restored progress of 2 [next = 345, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.574746 2403 raft/raft.go:1135 [s3,r1/?:/M{in-ax}] 39 restored progress of 39 [next = 345, match = 344, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.574760 2403 raft/raft.go:1102 [s3,r1/?:/M{in-ax}] 39 [commit: 344] restored snapshot [index: 344, term: 6]
I161110 14:11:19.574780 2403 storage/replica_raftstorage.go:589 [s3,r1/?:/M{in-ax}] applying Raft snapshot at index 344 (id=2419a90f, encoded size=16, 1 rocksdb batches, 5 log entries)
I161110 14:11:19.580488 2403 storage/replica_raftstorage.go:592 [s3,r1/57:/M{in-ax}] applied Raft snapshot in 0.006s
I161110 14:11:19.580709 2392 raft/raft.go:854 [s1,r1/1:/M{in-ax}] 1 no progress available for 39
I161110 14:11:19.580815 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 437218ed at index 345 in 51.188µs.
I161110 14:11:19.581591 274 storage/store.go:2984 [s3,r1/57:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.581907 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 470, log entries: 6
I161110 14:11:19.582081 2405 raft/raft.go:568 [s3,r1/57:/M{in-ax}] ffffffffffffffff became follower at term 6
I161110 14:11:19.582213 2405 raft/raft.go:316 [s3,r1/57:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,39], term: 6, commit: 344, applied: 344, lastindex: 344, lastterm: 6]
I161110 14:11:19.584676 2405 raft/raft.go:1125 [s3,r1/57:/M{in-ax}] ffffffffffffffff [commit: 344, lastindex: 344, lastterm: 6] starts to restore snapshot [index: 345, term: 6]
I161110 14:11:19.584710 2405 raft/log.go:288 [s3,r1/57:/M{in-ax}] log [committed=344, applied=344, unstable.offset=345, len(unstable.Entries)=0] starts to restore snapshot [index: 345, term: 6]
I161110 14:11:19.584741 2405 raft/raft.go:1135 [s3,r1/57:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 346, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.584762 2405 raft/raft.go:1135 [s3,r1/57:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 346, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.584774 2405 raft/raft.go:1102 [s3,r1/57:/M{in-ax}] ffffffffffffffff [commit: 345] restored snapshot [index: 345, term: 6]
I161110 14:11:19.584792 2405 storage/replica_raftstorage.go:589 [s3,r1/57:/M{in-ax}] applying Raft snapshot at index 345 (id=437218ed, encoded size=16, 1 rocksdb batches, 6 log entries)
I161110 14:11:19.590625 2405 storage/replica_raftstorage.go:592 [s3,r1/57:/M{in-ax}] applied Raft snapshot in 0.006s
I161110 14:11:19.591119 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:58
I161110 14:11:19.592250 2409 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:58}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:58}]
I161110 14:11:19.593768 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:58 > next_replica_id:59
I161110 14:11:19.595738 2441 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:58}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.596322 201 storage/store.go:2037 [s3,r1/57:/M{in-ax}] DEL 1
I161110 14:11:19.596617 76 raft/raft.go:568 [s3,r1/58:{-}] 3a became follower at term 0
I161110 14:11:19.596990 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 90693993 at index 349 in 33.737µs.
I161110 14:11:19.597865 76 raft/raft.go:316 [s3,r1/58:{-}] newRaft 3a [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.597885 76 raft/raft.go:568 [s3,r1/58:{-}] 3a became follower at term 1
I161110 14:11:19.597905 76 raft/raft.go:710 [s3,r1/58:{-}] 3a [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:19.597916 76 raft/raft.go:568 [s3,r1/58:{-}] 3a became follower at term 6
I161110 14:11:19.598124 76 raft/raft.go:1088 [s3,r1/58:{-}] 3a [logterm: 0, index: 347] rejected msgApp [logterm: 6, index: 347] from 1
I161110 14:11:19.599089 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 476, log entries: 10
I161110 14:11:19.599209 2452 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.599281 2452 raft/raft.go:568 [s3,r1/58:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.600606 2452 raft/raft.go:316 [s3,r1/58:{-}] newRaft ffffffffffffffff [peers: [], term: 6, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.600629 2452 raft/raft.go:568 [s3,r1/58:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.600651 2452 raft/raft.go:710 [s3,r1/58:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.600662 2452 raft/raft.go:568 [s3,r1/58:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.601866 2452 raft/raft.go:1125 [s3,r1/58:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 349, term: 6]
I161110 14:11:19.601890 2452 raft/log.go:288 [s3,r1/58:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 349, term: 6]
I161110 14:11:19.601910 2452 raft/raft.go:1135 [s3,r1/58:{-}] ffffffffffffffff restored progress of 1 [next = 350, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.601927 2452 raft/raft.go:1135 [s3,r1/58:{-}] ffffffffffffffff restored progress of 2 [next = 350, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.601938 2452 raft/raft.go:1102 [s3,r1/58:{-}] ffffffffffffffff [commit: 349] restored snapshot [index: 349, term: 6]
I161110 14:11:19.601954 2452 storage/replica_raftstorage.go:589 [s3,r1/58:{-}] applying Raft snapshot at index 349 (id=90693993, encoded size=16, 1 rocksdb batches, 10 log entries)
I161110 14:11:19.605910 2452 storage/replica_raftstorage.go:592 [s3,r1/58:/M{in-ax}] applied Raft snapshot in 0.004s
I161110 14:11:19.605933 2452 storage/store.go:2813 [s3,r1/58:/M{in-ax}] DEL 1
I161110 14:11:19.606111 274 storage/store.go:2984 [s3,r1/58:/M{in-ax}] added to replica GC queue (peer suggestion)
I161110 14:11:19.609570 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:59
I161110 14:11:19.615700 2472 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:59}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:59}]
I161110 14:11:19.618089 7 storage/replica_command.go:3244 [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:59 > next_replica_id:60
I161110 14:11:19.620157 2485 storage/replica.go:2052 [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:59}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161110 14:11:19.621527 7 storage/replica_raftstorage.go:445 [s1,r1/1:/M{in-ax}] generated snapshot 358cce60 at index 356 in 86.859µs.
I161110 14:11:19.623462 7 storage/store.go:3127 [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 485, log entries: 17
I161110 14:11:19.783785 201 storage/store.go:2037 [s3,r1/58:/M{in-ax}] DEL 1
I161110 14:11:19.784147 79 raft/raft.go:568 [s3,r1/59:{-}] 3b became follower at term 0
I161110 14:11:19.785359 79 raft/raft.go:316 [s3,r1/59:{-}] newRaft 3b [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.785383 79 raft/raft.go:568 [s3,r1/59:{-}] 3b became follower at term 1
I161110 14:11:19.785404 79 raft/raft.go:710 [s3,r1/59:{-}] 3b [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161110 14:11:19.785416 79 raft/raft.go:568 [s3,r1/59:{-}] 3b became follower at term 6
I161110 14:11:19.785430 79 raft/raft.go:1088 [s3,r1/59:{-}] 3b [logterm: 0, index: 354] rejected msgApp [logterm: 6, index: 354] from 1
I161110 14:11:19.785518 2457 storage/store.go:1908 [s3] ADD 1
I161110 14:11:19.785587 79 storage/store.go:3216 [s3] DEL 1
I161110 14:11:19.785625 2457 raft/raft.go:568 [s3,r1/59:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.786926 2457 raft/raft.go:316 [s3,r1/59:{-}] newRaft ffffffffffffffff [peers: [], term: 6, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161110 14:11:19.786970 2457 raft/raft.go:568 [s3,r1/59:{-}] ffffffffffffffff became follower at term 1
I161110 14:11:19.786991 2457 raft/raft.go:710 [s3,r1/59:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161110 14:11:19.787003 2457 raft/raft.go:568 [s3,r1/59:{-}] ffffffffffffffff became follower at term 6
I161110 14:11:19.791061 2457 raft/raft.go:1125 [s3,r1/59:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 356, term: 6]
I161110 14:11:19.791114 2457 raft/log.go:288 [s3,r1/59:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 356, term: 6]
I161110 14:11:19.791138 2457 raft/raft.go:1135 [s3,r1/59:{-}] ffffffffffffffff restored progress of 1 [next = 357, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.791154 2457 raft/raft.go:1135 [s3,r1/59:{-}] ffffffffffffffff restored progress of 2 [next = 357, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161110 14:11:19.791247 2457 raft/raft.go:1102 [s3,r1/59:{-}] ffffffffffffffff [commit: 356] restored snapshot [index: 356, term: 6]
I161110 14:11:19.791404 2457 storage/replica_raftstorage.go:589 [s3,r1/59:{-}] applying Raft snapshot at index 356 (id=358cce60, encoded size=16, 1 rocksdb batches, 17 log entries)
E161110 14:11:19.793767 274 storage/store.go:2982 [s3,r1/59:{-}] unable to add to replica GC queue: replica not initialized
I161110 14:11:19.807377 2457 storage/replica_raftstorage.go:592 [s3,r1/59:/M{in-ax}] applied Raft snapshot in 0.016s
F161110 14:11:19.807412 2457 storage/store.go:2811 [s3,r1/59:/M{in-ax}] could not remove placeholder after preemptive snapshot
goroutine 2457 [running]:
github.com/cockroachdb/cockroach/pkg/util/log.getStacks(0x901, 0x302022a8, 0x5f772a0, 0xc420bf9068)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:849 +0xa5
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).outputLogEntry(0x5f78800, 0xc400000004, 0x5b4411b, 0x10, 0xafb, 0xc4209b12c0, 0x4b)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:712 +0x7a6
github.com/cockroachdb/cockroach/pkg/util/log.addStructured(0x784a278, 0xc420849620, 0x4, 0x2, 0x53e4e48, 0x36, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/structured.go:140 +0x296
github.com/cockroachdb/cockroach/pkg/util/log.logDepth(0x784a278, 0xc420849620, 0x1, 0x4, 0x53e4e48, 0x36, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:88 +0x8c
github.com/cockroachdb/cockroach/pkg/util/log.Fatalf(0x784a278, 0xc420849620, 0x53e4e48, 0x36, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/log.go:172 +0x7e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest.func3(0xc420076380, 0xc420bf949b, 0xc42081d318, 0x784a278, 0xc420849620, 0xc420bf9b18, 0xc420bf949a, 0xc420bcf400)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2811 +0x28b
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420076380, 0x784a278, 0xc420849620, 0xc42081d318, 0x1a4edd3760ce8c35, 0x7a5dccc68b5c0e8f, 0x1, 0x5fad4c0, 0x0, 0x0, ...)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2921 +0x83c
github.com/cockroachdb/cockroach/pkg/storage.(*Store).HandleSnapshot(0xc420076380, 0xc42081d2c0, 0x76b9db0, 0xc4202b0b90, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2606 +0x746
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftSnapshot.func1.1(0x5c0fea0, 0xc4202b0b90, 0xc420587680, 0xc4202a0738, 0xc4202a0740)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:361 +0x17f
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftSnapshot.func1(0x784a278, 0xc42049af00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:362 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc42017e000, 0x5b441ce, 0x19, 0x16b, 0x0, 0x0, 0xc42049af90, 0x784a278, 0xc42049af00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234
goroutine 1 [chan receive]:
testing.(*T).Run(0xc42008e3c0, 0x53ab1e1, 0x12, 0x5578bc0, 0x4060201)
/Users/tamird/src/go1.7/src/testing/testing.go:647 +0x316
testing.RunTests.func1(0xc42008e3c0)
/Users/tamird/src/go1.7/src/testing/testing.go:793 +0x6d
testing.tRunner(0xc42008e3c0, 0xc420243bc0)
/Users/tamird/src/go1.7/src/testing/testing.go:610 +0x81
testing.RunTests(0x557a2b8, 0x5f6ab40, 0x156, 0x156, 0xc4202b0490)
/Users/tamird/src/go1.7/src/testing/testing.go:799 +0x2f5
testing.(*M).Run(0xc420243ee8, 0xc420247d30)
/Users/tamird/src/go1.7/src/testing/testing.go:743 +0x85
github.com/cockroachdb/cockroach/pkg/storage_test.TestMain(0xc420243ee8)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/main_test.go:57 +0x1ff
main.main()
github.com/cockroachdb/cockroach/pkg/storage/_test/_testmain.go:746 +0xc6
goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/Users/tamird/src/go1.7/src/runtime/asm_amd64.s:2086 +0x1
goroutine 34 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).flushDaemon(0x5f78800)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1001 +0x77
created by github.com/cockroachdb/cockroach/pkg/util/log.init.1
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:579 +0x95
goroutine 7 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420a05600, 0xc420ad0b30, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420385e00, 0xc420ad0b30, 0x5, 0x5, 0xffffffff, 0x0, 0x77)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc420385e00, 0xc420ad0b30, 0x5, 0x5, 0x5, 0xc4204a3ec0, 0x77, 0x113)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc420385e00, 0xc420ad0b30, 0x5, 0x5, 0x0, 0xc4204a3ec0, 0xc4204a3ec0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc420ad0b20, 0x7fffffff, 0x1485c56286ffed0f, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc420ad0b20, 0x5c08b80, 0x5fad4c0, 0xc420385e00, 0x0, 0x0, 0x52f4960, 0xc4208c0300, 0x7fffffff, 0xc42060c800, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc420ac2480, 0x52f4960, 0xc4208c0300, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftSnapshotClient).Recv(0xc420441830, 0x30, 0xc4209e3380, 0x2)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:309 +0x62
github.com/cockroachdb/cockroach/pkg/storage.snapshotClientWithBreaker.Recv(0x5c11280, 0xc420441830, 0xc4207fc510, 0xc4209e3380, 0x2, 0x2)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:604 +0x35
github.com/cockroachdb/cockroach/pkg/storage.(*snapshotClientWithBreaker).Recv(0xc420ad0b40, 0xc4202ffcb0, 0x53dc1bd, 0x30)
<autogenerated>:669 +0x64
github.com/cockroachdb/cockroach/pkg/storage.sendSnapshot(0x784a278, 0xc4202ffcb0, 0x5bff300, 0xc420ad0b40, 0x5bfd4c0, 0xc42017e1b0, 0x1, 0x5fad4c0, 0x0, 0x0, ...)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3129 +0xaf6
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendSnapshot(0xc420146420, 0x784a278, 0xc4202ffcb0, 0xc42017e1b0, 0x1, 0x5fad4c0, 0x0, 0x0, 0xc420441568, 0x2, ...)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:648 +0x2f3
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).ChangeReplicas.func1(0xc4201d1000, 0x784a278, 0xc4202ffcb0, 0xc4204415a0, 0xc4203b0780, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:3209 +0x4fc
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).ChangeReplicas(0xc4201d1000, 0x784a278, 0xc4202ffcb0, 0x300000000, 0x3, 0xc4203b0480, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_command.go:3213 +0x3d9
github.com/cockroachdb/cockroach/pkg/storage_test.(*multiTestContext).replicateRange(0xc42005e340, 0x1, 0xc4209e3f28, 0x1, 0x1)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_test.go:925 +0x3f6
github.com/cockroachdb/cockroach/pkg/storage_test.TestRaftRemoveRace(0xc42008e480)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_raft_test.go:2004 +0x14a
testing.tRunner(0xc42008e480, 0x5578bc0)
/Users/tamird/src/go1.7/src/testing/testing.go:610 +0x81
created by testing.(*T).Run
/Users/tamird/src/go1.7/src/testing/testing.go:646 +0x2ec
goroutine 36 [chan receive]:
github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:137 +0x76
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc420265b40)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 37 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc420114dc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:373 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:374 +0x702
goroutine 38 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*StorePool).start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store_pool.go:352 +0x3de
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc420208940)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 39 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc4201650e0, 0x78440d8, 0xc420012568)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:207 +0xbd3
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:191 +0x98
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc4202859b0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 40 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:49 +0x6f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201e3230)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 41 [IO wait]:
net.runtime_pollWait(0x784b620, 0x72, 0x0)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420206ae0, 0x72, 0xc42002a5f0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420206ae0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc420206a80, 0x0, 0x5bf8c00, 0xc4201ba180)
/Users/tamird/src/go1.7/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc420128290, 0x403674e, 0xc42002a6a0, 0x44b2429)
/Users/tamird/src/go1.7/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc420128290, 0x557a030, 0xc420164140, 0x5c113a0, 0xc420120048)
/Users/tamird/src/go1.7/src/net/tcpsock.go:222 +0x49
google.golang.org/grpc.(*Server).Serve(0xc420164140, 0x5c07c00, 0xc420128290, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:348 +0x15a
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:56 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42011f4e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 47 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420136000, 0xc420132c00, 0xc42011a870, 0xf, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x617
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201e3c20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 44 [IO wait]:
net.runtime_pollWait(0x784b560, 0x72, 0x6)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420206f40, 0x72, 0xc42050eb08, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420206f40, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc420206ee0, 0xc4200d6000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420128298, 0xc4200d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc42000cf60)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc42000cf60, 0xc42015e938, 0x9, 0x9, 0x4056700, 0xc420654a40, 0xc42050ed08)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc42000cf60, 0xc42015e938, 0x9, 0x9, 0x9, 0xc4201de548, 0x5bf7e40, 0xc4208106c0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc42000cf60, 0xc42015e938, 0x9, 0x9, 0x47bd578, 0xc4201c2690, 0xc420aa5000)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc42015e938, 0x9, 0x9, 0x5bf3480, 0xc42000cf60, 0x0, 0xc400000000, 0x0, 0xed7)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42015e900, 0x5bfdc40, 0xc420810690, 0x0, 0x0)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201e3b30, 0xc420810690, 0xc420810690, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc420132d00)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1
goroutine 22 [IO wait]:
net.runtime_pollWait(0x784b4a0, 0x72, 0x7)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42016e060, 0x72, 0xc4200428e8, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42016e060, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42016e000, 0xc4200e6000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420120048, 0xc4200e6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc4201600c0)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc4201600c0, 0xc42008e038, 0x9, 0x9, 0x557289d, 0x557289c, 0xc420348238)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc4201600c0, 0xc42008e038, 0x9, 0x9, 0x9, 0xc420042af8, 0x4056700, 0xc420186c80)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc4201600c0, 0xc42008e038, 0x9, 0x9, 0x47b4c32, 0xc420346184, 0xc420042b60)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc42008e038, 0x9, 0x9, 0x5bf3480, 0xc4201600c0, 0x0, 0xc400000000, 0xc420042c90, 0x47c27fc)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42008e000, 0x3a, 0x3a, 0x0, 0x0)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201e0090, 0xc4201e18c0, 0xc4201e18c0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42001e090, 0xc4201e0150)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200
google.golang.org/grpc.(*Server).serveStreams(0xc420164140, 0x5c100e0, 0xc42001e090)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:424 +0x160
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420164140, 0x5c113a0, 0xc420120048, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:411 +0x3da
google.golang.org/grpc.(*Server).handleRawConn(0xc420164140, 0x5c113a0, 0xc420120048)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:388 +0x470
created by google.golang.org/grpc.(*Server).Serve
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:357 +0x1a9
goroutine 45 [select]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420132d00)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78
goroutine 46 [select]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc420167040)
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2
goroutine 23 [select]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc42001e090)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c
created by google.golang.org/grpc/transport.newHTTP2Server
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767
goroutine 48 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42011f9e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 49 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:933 +0x46c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42011aaa0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 50 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:962 +0x394
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42011aab0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 68 [select]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420596000)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78
goroutine 51 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:171 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c5c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 52 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000667)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c5e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 53 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000666)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c600)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 54 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000662)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c620)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 55 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000660)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c640)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 56 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000663)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c660)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 57 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000665)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c680)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 58 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000664)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c6a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 59 [semacquire]:
sync.runtime_notifyListWait(0xc42012ce90, 0xc400000661)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42012ce80)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201654a0, 0xc42017e120)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c6c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 60 [semacquire]:
sync.runtime_Semacquire(0xc42016553c)
/Users/tamird/src/go1.7/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420165530)
/Users/tamird/src/go1.7/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4201654a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3251 +0x3a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42011ba80)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 61 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3277 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42011ba90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 62 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3308 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42011baa0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 64 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1142 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029c780)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 65 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc4201dcb10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 82 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc4201dcb40)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 115 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42029ccc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 97 [select]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc42009b040)
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2
goroutine 160 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420132d00, 0xc4201c2690, 0xc420092fc0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 94 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4201659a0, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0x6fc602f, 0x5f772a0, 0xc42012c080, 0xc42017e120, 0xc4201d1000, 0xecfb6bd51)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4201d1000, 0xc4201e2cc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc4201e2fc0, 0xc4204fe910)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:303 +0x3f5
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2ca
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cbc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 93 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cba0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 96 [select]:
google.golang.org/grpc/transport.(*http2Client).controller(0xc420133300)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:999 +0x59d
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:233 +0xe78
goroutine 87 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029caa0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 88 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cac0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 89 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cae0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 90 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cb00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 91 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cb20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 92 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc4201655e0, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0x6fb6b97, 0x5f772a0, 0xc42012c080, 0xc42017e120, 0xc4201d1000, 0xecfb6bd51)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc4201d1000, 0xc4201e2cc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:269 +0x8e
github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc4201e2db0, 0xc4204fe8c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:303 +0x3f5
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:271 +0x2ca
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc42029cb40)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 33 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*StorePool).start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store_pool.go:352 +0x3de
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4201bb600)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 32 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc42059a550)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:373 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:374 +0x702
goroutine 31 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:144 +0x465
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e120, 0xc4201bb2c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 98 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc420099680, 0x78440d8, 0xc420012568)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:207 +0xbd3
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:191 +0x98
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc420594d00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 99 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:49 +0x6f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42015af00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 100 [IO wait]:
net.runtime_pollWait(0x784b3e0, 0x72, 0x0)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42006a8b0, 0x72, 0xc4205815f0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42006a8b0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42006a850, 0x0, 0x5bf8c00, 0xc4200ce8c0)
/Users/tamird/src/go1.7/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc420120238, 0x403674e, 0xc4205816a0, 0x44b2429)
/Users/tamird/src/go1.7/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc420120238, 0x557a030, 0xc420098820, 0x5c113a0, 0xc420656000)
/Users/tamird/src/go1.7/src/net/tcpsock.go:222 +0x49
google.golang.org/grpc.(*Server).Serve(0xc420098820, 0x5c07c00, 0xc420120238, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:348 +0x15a
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:56 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201bbf60)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 114 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420136000, 0xc420596e00, 0xc4205957a0, 0xf, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x617
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201dd5f0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 95 [IO wait]:
net.runtime_pollWait(0x784b320, 0x72, 0x9)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42006aa70, 0x72, 0xc420508b08, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42006aa70, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42006aa10, 0xc4205fc000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420128330, 0xc4205fc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc420586240)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420586240, 0xc42015ee78, 0x9, 0x9, 0x10, 0xc4208987e0, 0x0)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc420586240, 0xc42015ee78, 0x9, 0x9, 0x9, 0x52e8f40, 0x4003701, 0xc4208987e0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc420586240, 0xc42015ee78, 0x9, 0x9, 0x426d238ff501, 0xbe00000000000004, 0x1)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc42015ee78, 0x9, 0x9, 0x5bf3480, 0xc420586240, 0x0, 0xc400000000, 0x402d2f1, 0xc420508e18)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42015ee40, 0x426d05bfdf00, 0x400000801, 0x80100000005, 0x500000004)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201dd530, 0xc4208987e0, 0xc4208987e0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc420133300)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1
goroutine 116 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:933 +0x46c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4202b06d0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 117 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:962 +0x394
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4202b06e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 118 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:171 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cd20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 119 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000415)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cd40)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 120 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000411)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cd60)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 121 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000412)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cd80)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 122 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000413)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cda0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 123 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000416)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cdc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 124 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000414)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cde0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 125 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000417)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029ce00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 126 [semacquire]:
sync.runtime_notifyListWait(0xc4205b3710, 0xc400000418)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc4205b3700)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420099860, 0xc42001ee10)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029ce20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 127 [semacquire]:
sync.runtime_Semacquire(0xc4200998fc)
/Users/tamird/src/go1.7/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc4200998f0)
/Users/tamird/src/go1.7/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420099860)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3251 +0x3a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4202b0760)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 128 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3277 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4202b0770)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 129 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3308 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4202b0780)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 131 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1142 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029cea0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 132 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4201dd740)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 133 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4201dd770)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 194 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4205b2500, 0xc4202084f0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4201c2690, 0xc4202084f0, 0x5, 0x5, 0xc420730bc0, 0x4496b42, 0xc420730c80)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4201c2690, 0xc4202084f0, 0x5, 0x5, 0x5, 0xc420a8fd40, 0x5, 0xc420730c58)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4201c2690, 0xc4202084f0, 0x5, 0x5, 0x5f86d80, 0x5f86d98, 0xc420730c70)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4202084e0, 0x7fffffff, 0xc400000001, 0x5fae2e0, 0x0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4202084e0, 0x5c08b80, 0x5fad4c0, 0xc4201c2690, 0x0, 0x0, 0x52ec8c0, 0xc4209f4050, 0x7fffffff, 0xc42005c000, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc420092fc0, 0x52ec8c0, 0xc4209f4050, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc420284f90, 0x784a200, 0xc4205b2440, 0xc420122480)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:192 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x5c10620, 0xc420284f90, 0xc4200a8c60, 0x784a200, 0xc4205b2440, 0xc420122480, 0x402c533, 0xc420614748)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:297 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:305 +0xd7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42059aeb0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 151 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cf9a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 137 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc4200a8580, 0x784a200, 0xc42065a240, 0xc420598180, 0x5c10620, 0xc4202b0c50, 0xc42017e000, 0xc4202b0c10, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:309 +0x3ec
github.com/cockroachdb/cockroach/pkg/gossip.(*client).start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:126 +0x4f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42065a1c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 10 [IO wait]:
net.runtime_pollWait(0x784b260, 0x72, 0xa)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc420142140, 0x72, 0xc4206668e8, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc420142140, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc4201420e0, 0xc42066c000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420656000, 0xc42066c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc42052e0c0)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc42052e0c0, 0xc420658038, 0x9, 0x9, 0x557289d, 0x557289c, 0xc42052f438)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc42052e0c0, 0xc420658038, 0x9, 0x9, 0x9, 0xc420666af8, 0x4056700, 0xc42021f0c0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc42052e0c0, 0xc420658038, 0x9, 0x9, 0x47b4c32, 0xc42012aea4, 0xc420666b60)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc420658038, 0x9, 0x9, 0x5bf3480, 0xc42052e0c0, 0x0, 0xc400000000, 0xc420666c90, 0x47c27fc)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420658000, 0x3b, 0x3b, 0x0, 0x0)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201b8900, 0xc4201e13e0, 0xc4201e13e0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42067c000, 0xc4201b9140)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200
google.golang.org/grpc.(*Server).serveStreams(0xc420098820, 0x5c100e0, 0xc42067c000)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:424 +0x160
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420098820, 0x5c113a0, 0xc420656000, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:411 +0x3da
google.golang.org/grpc.(*Server).handleRawConn(0xc420098820, 0x5c113a0, 0xc420656000)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:388 +0x470
created by google.golang.org/grpc.(*Server).Serve
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:357 +0x1a9
goroutine 11 [select]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc42067c000)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c
created by google.golang.org/grpc/transport.newHTTP2Server
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767
goroutine 13 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc42015e180, 0x5c0fb40, 0xc420292a80, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x57b
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x5354840, 0xc42015e180, 0x5c0c600, 0xc42067c090, 0xc420142288, 0xc4202a67a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:209 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4206880f0, 0xc4201e2360, 0x5f576c0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4206880f0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4200122d0, 0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4206880f0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 138 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420132d00, 0xc4201c2870, 0xc4200925a0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 159 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc4200a8c60, 0x784a200, 0xc4205b2440, 0xc420122480, 0x5c10620, 0xc420284f90, 0xc42017e000, 0xc420284ea0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:309 +0x3ec
github.com/cockroachdb/cockroach/pkg/gossip.(*client).start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:126 +0x4f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4205b23c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 140 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42065a300, 0xc42029d350, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4201c2870, 0xc42029d350, 0x5, 0x5, 0xc420514bc0, 0x4496b42, 0xc420514c80)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4201c2870, 0xc42029d350, 0x5, 0x5, 0x5, 0x48cf70ffd8a026aa, 0x0, 0xc420012580)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4201c2870, 0xc42029d350, 0x5, 0x5, 0x48850c7, 0xc420594870, 0x1)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc42029d340, 0x7fffffff, 0xc400000001, 0x5fae2e0, 0x0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc42029d340, 0x5c08b80, 0x5fad4c0, 0xc4201c2870, 0x0, 0x0, 0x52ec8c0, 0xc4209b0ff0, 0x7fffffff, 0xc420033400, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc4200925a0, 0x52ec8c0, 0xc4209b0ff0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc4202b0c50, 0x784a200, 0xc42065a240, 0xc420598180)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:192 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x5c10620, 0xc4202b0c50, 0xc4200a8580, 0x784a200, 0xc42065a240, 0xc420598180, 0x402c533, 0xc420529748)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:297 +0x35
github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:305 +0xd7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4204ff6d0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 14 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42021f300, 0xc4200cedd0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4206880f0, 0xc4200cedd0, 0x5, 0x5, 0xc420668948, 0x4042afb, 0x30)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4206880f0, 0xc4200cedd0, 0x5, 0x5, 0x5, 0xc420682d50, 0x0, 0xc420682d80)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4206880f0, 0xc4200cedd0, 0x5, 0x5, 0xc42052e538, 0x1, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4200cedc0, 0x7fffffff, 0x0, 0x38, 0x55728df, 0x55728de, 0xc41ffef033, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4200cedc0, 0x5c08b80, 0x5fad4c0, 0xc4206880f0, 0x0, 0x0, 0x52ff5e0, 0xc42021f940, 0x7fffffff, 0x20001, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc42067c090, 0x52ff5e0, 0xc42021f940, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc420292a80, 0x0, 0x0, 0x100010000)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:228 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc42015e1b0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x2f
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc42015e180, 0x784a200, 0xc42021f380, 0xc420656008, 0xc42021f3c0, 0xc420668f50, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:321 +0x413
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x99
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42051eaa0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 141 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:144 +0x465
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc42029d5a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 153 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cf9e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 152 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cf9c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 150 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420099ae0, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0xc4074ceab1, 0x5f772a0, 0xc42012c080, 0xc42001ee10, 0x0, 0x5579148)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cf940)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 149 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cf920)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 154 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cfa00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 274 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4205b3a40, 0xc4203fe270, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4201c30e0, 0xc4203fe270, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4201c30e0, 0xc4203fe270, 0x5, 0x5, 0x5, 0x112, 0x5b4411b, 0x10)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4201c30e0, 0xc4203fe270, 0x5, 0x5, 0x5b4411b, 0x10, 0xba6)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4203fe260, 0x7fffffff, 0xc420936a20, 0x51c5b40, 0xc420936a30, 0x51c6580, 0xc420936a40, 0xc4203efd08)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4203fe260, 0x5c08b80, 0x5fad4c0, 0xc4201c30e0, 0x0, 0x0, 0x52f45e0, 0xc4203baed0, 0x7fffffff, 0xc420033400, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc420093c20, 0x52f45e0, 0xc4203baed0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42011b9a0, 0x784a278, 0xc4203bade0, 0xc4204c1e90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x5c11220, 0xc42011b9a0, 0xc4205b2e80, 0xc420587680, 0x78440d8, 0xc420012568, 0x450385f, 0xc4202a7788)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:477 +0xdd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4203de900)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 155 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cfa20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 156 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420099900, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0xc4074d557e, 0x5f772a0, 0xc42012c080, 0xc42001ee10, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42001ee10, 0xc4200cfa40)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 163 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:144 +0x465
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc42011e3a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 142 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open.func1(0xc42069a190)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:373 +0x44
created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:374 +0x702
goroutine 143 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*StorePool).start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store_pool.go:352 +0x3de
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc42029d8c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 144 [select]:
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).printStatsLoop(0xc4205310e0, 0x78440d8, 0xc420012568)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:207 +0xbd3
github.com/cockroachdb/cockroach/pkg/kv.NewTxnCoordSender.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:191 +0x98
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4202b1330)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 165 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4200167c0, 0xc42011e570, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420688000, 0xc42011e570, 0x5, 0x5, 0xc42072e948, 0x4042afb, 0x40)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc420688000, 0xc42011e570, 0x5, 0x5, 0x5, 0xc4205b2780, 0x0, 0xc4205b27c0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc420688000, 0xc42011e570, 0x5, 0x5, 0xc42000d258, 0x1, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc42011e560, 0x7fffffff, 0x0, 0x38, 0x55728df, 0x55728de, 0xc41ffd26bd, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc42011e560, 0x5c08b80, 0x5fad4c0, 0xc420688000, 0x0, 0x0, 0x52ff5e0, 0xc4205b2800, 0x7fffffff, 0x20001, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc42001e120, 0x52ff5e0, 0xc4205b2800, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc4202004a0, 0x0, 0x0, 0x1000000010000)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:228 +0x62
github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc42015e1b0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x2f
github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc42015e180, 0x784a200, 0xc4200168c0, 0xc420656010, 0xc420016980, 0xc42072ef50, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:321 +0x413
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:153 +0x99
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc420510dc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 164 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc42015e180, 0x5c0fb40, 0xc4202004a0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:190 +0x57b
github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x5354840, 0xc42015e180, 0x5c0c600, 0xc42001e120, 0x1a, 0x4013ab6)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:209 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc420688000, 0xc4201e2360, 0x5f576c0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc420688000, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4200122d0, 0xc420164140, 0x5c100e0, 0xc42001e090, 0xc420688000)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 110 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:49 +0x6f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201e2000)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 111 [IO wait]:
net.runtime_pollWait(0x784b1a0, 0x72, 0x0)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42026c8b0, 0x72, 0xc4206c9de0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42026c8b0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).accept(0xc42026c850, 0x0, 0x5bf8c00, 0xc42029c820)
/Users/tamird/src/go1.7/src/net/fd_unix.go:419 +0x238
net.(*TCPListener).accept(0xc4200300c8, 0x403674e, 0xc4206c9e90, 0x44b2429)
/Users/tamird/src/go1.7/src/net/tcpsock_posix.go:132 +0x2e
net.(*TCPListener).Accept(0xc4200300c8, 0x557a030, 0xc420530960, 0x5c113a0, 0xc420030198)
/Users/tamird/src/go1.7/src/net/tcpsock.go:222 +0x49
google.golang.org/grpc.(*Server).Serve(0xc420530960, 0x5c07c00, 0xc4200300c8, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:348 +0x15a
github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:56 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42029c700)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 70 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420136000, 0xc42000b700, 0xc420124ea0, 0xf, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:291 +0x617
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:226 +0x69
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201e03f0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 67 [IO wait]:
net.runtime_pollWait(0x784b0e0, 0x72, 0xc)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42026cd10, 0x72, 0xc4206c8b08, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42026cd10, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42026ccb0, 0xc420588000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420120140, 0xc420588000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc420160660)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420160660, 0xc420532038, 0x9, 0x9, 0x10, 0xc4202b0da0, 0x0)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc420160660, 0xc420532038, 0x9, 0x9, 0x9, 0x52e8f40, 0x4003701, 0xc4202b0da0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc420160660, 0xc420532038, 0x9, 0x9, 0x5c60238ff501, 0xbe00000000000004, 0x1)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc420532038, 0x9, 0x9, 0x5bf3480, 0xc420160660, 0x0, 0xc400000000, 0x402d2f1, 0xc4206c8e18)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc420532000, 0x5c6005bfdf00, 0x400000801, 0x80100000095, 0x9500000004)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201e0210, 0xc4202b0da0, 0xc4202b0da0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Client).reader(0xc420596000)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:921 +0xb6
created by google.golang.org/grpc/transport.newHTTP2Client
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_client.go:203 +0x8b1
goroutine 69 [select]:
google.golang.org/grpc.(*addrConn).transportMonitor(0xc420166280)
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:720 +0x63e
created by google.golang.org/grpc.(*ClientConn).resetAddrConn
/Users/tamird/src/go/src/google.golang.org/grpc/clientconn.go:467 +0x6e2
goroutine 71 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:381 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4201ba260)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 72 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:933 +0x46c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4205940a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 73 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:962 +0x394
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4205940b0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 74 [chan receive]:
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:171 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba2e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 75 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000206)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba300)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 76 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000205)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba320)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 77 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000202)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba340)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 78 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000203)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba360)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 79 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000208)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba380)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 80 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000204)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba3a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 81 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000201)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba3e0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 178 [semacquire]:
sync.runtime_notifyListWait(0xc42011c990, 0xc400000207)
/Users/tamird/src/go1.7/src/runtime/sema.go:267 +0x122
sync.(*Cond).Wait(0xc42011c980)
/Users/tamird/src/go1.7/src/sync/cond.go:57 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4201643c0, 0xc42017fb90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:212 +0xb2
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba420)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 179 [semacquire]:
sync.runtime_Semacquire(0xc42016445c)
/Users/tamird/src/go1.7/src/runtime/sema.go:47 +0x30
sync.(*WaitGroup).Wait(0xc420164450)
/Users/tamird/src/go1.7/src/sync/waitgroup.go:131 +0x97
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4201643c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:187 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaft.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3251 +0x3a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc420594180)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 180 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3277 +0x334
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc420594190)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 181 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startCoalescedHeartbeatsLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3308 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4205941a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 183 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func3()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1142 +0x165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201ba4c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 184 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201e06c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 185 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1238 +0x39b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4201e07b0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 206 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420587680, 0x5c0fe40, 0xc42011ac40, 0xc4202a24a8, 0x20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x536e620, 0xc420587680, 0x5c0c600, 0xc420520480, 0x42fa1d9, 0xc4201ba204)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4201c2f00, 0xc420296ed0, 0x5f5e780, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4201c2f00, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420124f70, 0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4201c2f00)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 198 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf780)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 203 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420164640, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0xc407929354, 0x5f772a0, 0xc42012c080, 0xc42017fb90, 0x0, 0x5579148)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf860)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 175 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc4201610e0, 0x5c0fe40, 0xc420680620, 0xc4205834a8, 0x20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x536e620, 0xc4201610e0, 0x5c0c600, 0xc42001f440, 0xc420583548, 0x402c533)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420098820, 0x5c100e0, 0xc42067c000, 0xc420688b40, 0xc420158ae0, 0x5f5e780, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420098820, 0x5c100e0, 0xc42067c000, 0xc420688b40, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420292280, 0xc420098820, 0x5c100e0, 0xc42067c000, 0xc420688b40)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 210 [IO wait]:
net.runtime_pollWait(0x784b020, 0x72, 0xd)
/Users/tamird/src/go1.7/src/runtime/netpoll.go:160 +0x59
net.(*pollDesc).wait(0xc42026ce60, 0x72, 0xc4206d28e8, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:73 +0x38
net.(*pollDesc).waitRead(0xc42026ce60, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_poll_runtime.go:78 +0x34
net.(*netFD).Read(0xc42026ce00, 0xc420794000, 0x8000, 0x8000, 0x0, 0x5bfbac0, 0xc4200120d8)
/Users/tamird/src/go1.7/src/net/fd_unix.go:243 +0x1a1
net.(*conn).Read(0xc420030198, 0xc420794000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/net/net.go:173 +0x70
bufio.(*Reader).fill(0xc420587f80)
/Users/tamird/src/go1.7/src/bufio/bufio.go:97 +0x10c
bufio.(*Reader).Read(0xc420587f80, 0xc42015e038, 0x9, 0x9, 0x4042e20, 0xc4202b0d50, 0x0)
/Users/tamird/src/go1.7/src/bufio/bufio.go:209 +0x1bc
io.ReadAtLeast(0x5bf3480, 0xc420587f80, 0xc42015e038, 0x9, 0x9, 0x9, 0xc4206d2af8, 0x4056700, 0xc420497540)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf3480, 0xc420587f80, 0xc42015e038, 0x9, 0x9, 0xc4202b0d50, 0x0, 0x1)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
golang.org/x/net/http2.readFrameHeader(0xc42015e038, 0x9, 0x9, 0x5bf3480, 0xc420587f80, 0x0, 0xc400000000, 0xc4206d2c90, 0x47c27fc)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:237 +0x7b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc42015e000, 0x2cb0, 0x2cb0, 0x0, 0x0)
/Users/tamird/src/go/src/golang.org/x/net/http2/frame.go:464 +0xa4
google.golang.org/grpc/transport.(*framer).readFrame(0xc4201e3110, 0xc42049b290, 0xc42049b290, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http_util.go:505 +0x2f
google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc42067c120, 0xc4201e31d0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:260 +0x200
google.golang.org/grpc.(*Server).serveStreams(0xc420530960, 0x5c100e0, 0xc42067c120)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:424 +0x160
google.golang.org/grpc.(*Server).serveNewHTTP2Transport(0xc420530960, 0x5c113a0, 0xc420030198, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:411 +0x3da
google.golang.org/grpc.(*Server).handleRawConn(0xc420530960, 0x5c113a0, 0xc420030198)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:388 +0x470
created by google.golang.org/grpc.(*Server).Serve
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:357 +0x1a9
goroutine 202 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf840)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 201 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf820)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 200 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf7c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 199 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf7a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 197 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420164e60, 0x78440d8, 0xc420012568, 0xecfb6bd51, 0xc4079237c0, 0x5f772a0, 0xc42012c080, 0xc42017fb90, 0x0, 0x5579148)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:207 +0x254
github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x3ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf6a0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 196 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:451 +0x357
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017fb90, 0xc4200cf680)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 209 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420132d00, 0xc4201c30e0, 0xc420093c20)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 208 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420587680, 0xc400000001, 0xc4206f4ea0, 0xc4205b2e80, 0x5c11220, 0xc42011b9a0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:488 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:434 +0x3be
github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4205207e0, 0xc4204001e0, 0x0, 0x784c2d8, 0xc42011ac40)
/Users/tamird/src/go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420587680, 0x78440d8, 0xc420012568, 0x1, 0xc4206f4ea0, 0xc4205b2e80)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:435 +0x161
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:568 +0xd6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4203de7b0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 211 [select]:
google.golang.org/grpc/transport.(*http2Server).controller(0xc42067c120)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:673 +0x69c
created by google.golang.org/grpc/transport.newHTTP2Server
/Users/tamird/src/go/src/google.golang.org/grpc/transport/http2_server.go:139 +0x767
goroutine 244 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4203288c0, 0xc4203460f0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4200d1e00, 0xc4203460f0, 0x5, 0x5, 0x0, 0x0, 0x4882c57)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4200d1e00, 0xc4203460f0, 0x5, 0x5, 0x5, 0x42faf09, 0xc420018410, 0xc400000001)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4200d1e00, 0xc4203460f0, 0x5, 0x5, 0x402d101, 0xc4202a1498, 0x457d0ad)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4203460e0, 0x7fffffff, 0xc42015aed0, 0x5bfe1c0, 0xc42015aed0, 0x0, 0x0, 0xc4206b6820)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4203460e0, 0x5c08b80, 0x5fad4c0, 0xc4200d1e00, 0x0, 0x0, 0x52f45e0, 0xc420134b40, 0x7fffffff, 0xc42060c800, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc4202317a0, 0x52f45e0, 0xc420134b40, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4203210f0, 0xc4202a1710, 0x40ba4b1, 0xc42001f320)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x5c11220, 0xc4203210f0, 0xc420328800, 0xc420146420, 0x78440d8, 0xc420012568, 0x450385f, 0xc4202a1788)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:477 +0xdd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42032e3c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 207 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4205b2d00, 0xc4200cfef0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4201c2f00, 0xc4200cfef0, 0x5, 0x5, 0xc42011c994, 0xc4207a9c10, 0x42f9e43)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4201c2f00, 0xc4200cfef0, 0x5, 0x5, 0x5, 0xc4207a9d88, 0x4117ba6, 0xc4201643c0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4201c2f00, 0xc4200cfef0, 0x5, 0x5, 0xc420823fa0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4200cfee0, 0x7fffffff, 0x1, 0x18, 0x5572885, 0x5572884, 0xc41ffa006b, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4200cfee0, 0x5c08b80, 0x5fad4c0, 0xc4201c2f00, 0x0, 0x0, 0x52f4500, 0xc420bff260, 0x7fffffff, 0xc42011ac01, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420520480, 0x52f4500, 0xc420bff260, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc42011ac40, 0x784a278, 0xc4203de120, 0xc4208dc600)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x5c0fe40, 0xc42011ac40, 0xc420587680, 0xc4202a7788, 0xc4202a7790)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4203de1b0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 246 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4203289c0, 0xc420346210, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4200d1ef0, 0xc420346210, 0x5, 0x5, 0xc42012ce94, 0xc420697c10, 0x42f9e43)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4200d1ef0, 0xc420346210, 0x5, 0x5, 0x5, 0xc420697d88, 0x4117ba6, 0xc4201654a0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4200d1ef0, 0xc420346210, 0x5, 0x5, 0xc4208c1ae0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc420346200, 0x7fffffff, 0x1, 0x18, 0x5572885, 0x5572884, 0xc41ffb9f09, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc420346200, 0x5c08b80, 0x5fad4c0, 0xc4200d1ef0, 0x0, 0x0, 0x52f4500, 0xc4208c1ea0, 0x7fffffff, 0xc420321101, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc4207fdd40, 0x52f4500, 0xc4208c1ea0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420321150, 0x784a278, 0xc42032e5a0, 0xc4209ebb00)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x5c0fe40, 0xc420321150, 0xc420146420, 0xc4202a4f88, 0xc4202a4f90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42032e5d0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 243 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420133300, 0xc4200d1e00, 0xc4202317a0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 235 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420329280, 0xc4206fa970, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc4203841e0, 0xc4206fa970, 0x5, 0x5, 0xc42012ce94, 0xc4207a6c10, 0x42f9e43)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc4203841e0, 0xc4206fa970, 0x5, 0x5, 0x5, 0xc4207a6d88, 0x4117ba6, 0xc4201654a0)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc4203841e0, 0xc4206fa970, 0x5, 0x5, 0xc4203feb60, 0x0, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4206fa960, 0x7fffffff, 0x1, 0x18, 0x5572885, 0x5572884, 0xc41fff22db, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4206fa960, 0x5c08b80, 0x5fad4c0, 0xc4203841e0, 0x0, 0x0, 0x52f4500, 0xc4201ba460, 0x7fffffff, 0xc420595e01, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc420316090, 0x52f4500, 0xc4201ba460, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420595e90, 0x784a278, 0xc42032f800, 0xc420bbad80)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x5c0fe40, 0xc420595e90, 0xc420146420, 0xc420526788, 0xc420526790)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4207ff200)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 245 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420146420, 0x5c0fe40, 0xc420321150, 0xc4202a4ca8, 0x20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x536e620, 0xc420146420, 0x5c0c600, 0xc4207fdd40, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4200d1ef0, 0xc420204840, 0x5f5e780, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4200d1ef0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4200122d0, 0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4200d1ef0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 242 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420146420, 0x2, 0xc4207f1e60, 0xc420328800, 0x5c11220, 0xc4203210f0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:488 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:434 +0x3be
github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc42001f320, 0xc420311400, 0x0, 0x5f772a0, 0xc4201d1000)
/Users/tamird/src/go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420146420, 0x78440d8, 0xc420012568, 0x2, 0xc4207f1e60, 0xc420328800)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:435 +0x161
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:568 +0xd6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42032e270)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 176 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4206f8680, 0xc42012af30, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420688b40, 0xc42012af30, 0x5, 0x5, 0xc4205b3714, 0xc420791c10, 0x42f9e43)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc420688b40, 0xc42012af30, 0x5, 0x5, 0x5, 0xc420791d88, 0x4117ba6, 0xc420099860)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc420688b40, 0xc42012af30, 0x5, 0x5, 0xc4208c1aa0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc42012af20, 0x7fffffff, 0x1, 0x18, 0x5572885, 0x5572884, 0xc41ffb9f0d, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc42012af20, 0x5c08b80, 0x5fad4c0, 0xc420688b40, 0x0, 0x0, 0x52f4500, 0xc4208c1e20, 0x7fffffff, 0xc420680601, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*serverStream).RecvMsg(0xc42001f440, 0x52f4500, 0xc4208c1e20, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420680620, 0x784a278, 0xc42073d0b0, 0xc4209eb980)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x5c0fe40, 0xc420680620, 0xc4201610e0, 0xc420583788, 0xc420583790)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:300 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:323 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42073d170)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 177 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4201610e0, 0xc400000001, 0xc42052f620, 0xc4206f8700, 0x5c11220, 0xc4206809a0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:488 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:434 +0x3be
github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc42001f7a0, 0xc4203502d0, 0x0, 0x784c2d8, 0xc420680620)
/Users/tamird/src/go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4201610e0, 0x78440d8, 0xc420012568, 0x1, 0xc42052f620, 0xc4206f8700)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:435 +0x161
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:568 +0xd6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42073d740)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 258 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420132d00, 0xc420688d20, 0xc42014a6c0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 259 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4206f88c0, 0xc42012b2f0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420688d20, 0xc42012b2f0, 0x5, 0x5, 0xc420583418, 0x0, 0xc420680598)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc420688d20, 0xc42012b2f0, 0x5, 0x5, 0x5, 0x0, 0xc420583430, 0x42d8034)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc420688d20, 0xc42012b2f0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc42012b2e0, 0x7fffffff, 0xc42012af30, 0x5, 0x5, 0x44b5a62, 0x536e620, 0xc4201610e0)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc42012b2e0, 0x5c08b80, 0x5fad4c0, 0xc420688d20, 0x0, 0x0, 0x52f45e0, 0xc42073d8f0, 0x7fffffff, 0xc42005c400, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc42014a6c0, 0x52f45e0, 0xc42073d8f0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4206809a0, 0xc42001f440, 0x52f4500, 0xc42012af80)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x5c11220, 0xc4206809a0, 0xc4206f8700, 0xc4201610e0, 0x78440d8, 0xc420012568, 0x450385f, 0xc420583788)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:477 +0xdd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc42073d8c0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 2456 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftSnapshot(0xc420587680, 0x5c0fea0, 0xc4202b0b90, 0xc4202a04a8, 0x20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:366 +0x23b
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftSnapshot_Handler(0x536e620, 0xc420587680, 0x5c0c600, 0xc4209acc60, 0x2, 0x2)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:353 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4200d0870, 0xc420296ed0, 0x5f5e7a0, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4200d0870, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc420124f70, 0xc420530960, 0x5c100e0, 0xc42067c120, 0xc4200d0870)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 264 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420146420, 0x3, 0xc4203b0ea0, 0xc4206f9dc0, 0x5c11220, 0xc4203bc6b0, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:488 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:434 +0x3be
github.com/rubyist/circuitbreaker.(*Breaker).Call(0xc4207fc510, 0xc4203c0190, 0x0, 0x784c2d8, 0xc420680620)
/Users/tamird/src/go/src/github.com/rubyist/circuitbreaker/circuitbreaker.go:339 +0x6b
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420146420, 0x78440d8, 0xc420012568, 0x3, 0xc4203b0ea0, 0xc4206f9dc0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:435 +0x161
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:568 +0xd6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4203bae70)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 265 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420596000, 0xc420689a40, 0xc4202c4120)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
goroutine 266 [select]:
google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4206f9e80, 0xc4203ab0d0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:141 +0x69a
google.golang.org/grpc/transport.(*Stream).Read(0xc420689a40, 0xc4203ab0d0, 0x5, 0x5, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x5bf7d00, 0xc420689a40, 0xc4203ab0d0, 0x5, 0x5, 0x5, 0x10a, 0x5b4411b, 0x10)
/Users/tamird/src/go1.7/src/io/io.go:307 +0xa4
io.ReadFull(0x5bf7d00, 0xc420689a40, 0xc4203ab0d0, 0x5, 0x5, 0x5b4411b, 0x10, 0xbac)
/Users/tamird/src/go1.7/src/io/io.go:325 +0x58
google.golang.org/grpc.(*parser).recvMsg(0xc4203ab0c0, 0x7fffffff, 0xc420200e60, 0x51c5b40, 0xc420200e38, 0x51c6580, 0xc420200e80, 0xc4207cbd08)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:231 +0x6f
google.golang.org/grpc.recv(0xc4203ab0c0, 0x5c08b80, 0x5fad4c0, 0xc420689a40, 0x0, 0x0, 0x52f45e0, 0xc4207e4330, 0x7fffffff, 0xc42005c400, ...)
/Users/tamird/src/go/src/google.golang.org/grpc/rpc_util.go:315 +0x4d
google.golang.org/grpc.(*clientStream).RecvMsg(0xc4202c4120, 0x52f45e0, 0xc4207e4330, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4203bc6b0, 0x784a278, 0xc4207e4300, 0xc420acede0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x5c11220, 0xc4203bc6b0, 0xc4206f9dc0, 0xc420146420, 0x78440d8, 0xc420012568, 0x450385f, 0xc4202a0f88)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:477 +0xdd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42017e000, 0xc4203baf90)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
goroutine 249 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420146420, 0x5c0fe40, 0xc420595e90, 0xc4202a4ca8, 0x20)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:329 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x536e620, 0xc420146420, 0x5c0c600, 0xc420316090, 0x0, 0x0)
/Users/tamird/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
google.golang.org/grpc.(*Server).processStreamingRPC(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4203841e0, 0xc420204840, 0x5f5e780, 0x0, 0x0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:689 +0x702
google.golang.org/grpc.(*Server).handleStream(0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4203841e0, 0x0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:773 +0xc36
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4200122d0, 0xc420164140, 0x5c100e0, 0xc42001e090, 0xc4203841e0)
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:422 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
/Users/tamird/src/go/src/google.golang.org/grpc/server.go:423 +0xa3
goroutine 2488 [select]:
google.golang.org/grpc.NewClientStream.func2(0x5c10080, 0xc420596000, 0xc420385e00, 0xc420ac2480)
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:197 +0x426
created by google.golang.org/grpc.NewClientStream
/Users/tamird/src/go/src/google.golang.org/grpc/stream.go:217 +0xb5f
ERROR: exit status 255
49 runs completed, 1 failures, over 54s
FAIL
make: *** [stress] Error 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment