test.log

$ make stress STRESSFLAGS='-p 100 -stderr -maxfails 1' PKG=./pkg/storage TESTS=TestRaftRemoveRace TESTFLAGS=--vmodule=raft=4,raft_log_queue=1 | tee test.log
go list -tags '' -f \
'go test -v  -tags '\'''\'' -ldflags '\'''\'' -i -c {{.ImportPath}} -o {{.Dir}}/stress.test && (cd {{.Dir}} && if [ -f stress.test ]; then stress -p 100 -stderr -maxfails 1 ./stress.test -test.run '\''TestRaftRemoveRace'\'' -test.timeout 2m --vmodule=raft=4,raft_log_queue=1; fi)' ./pkg/storage | \
/bin/bash
github.com/cockroachdb/cockroach/pkg/storage
github.com/cockroachdb/cockroach/pkg/storage_test
testmain
0 runs so far, 0 failures, over 5s
0 runs so far, 0 failures, over 10s
0 runs so far, 0 failures, over 15s
0 runs so far, 0 failures, over 20s
0 runs so far, 0 failures, over 25s
0 runs so far, 0 failures, over 30s
0 runs so far, 0 failures, over 35s
0 runs so far, 0 failures, over 40s
0 runs so far, 0 failures, over 45s
0 runs so far, 0 failures, over 50s
0 runs so far, 0 failures, over 55s
0 runs so far, 0 failures, over 1m0s
0 runs so far, 0 failures, over 1m5s
6 runs so far, 0 failures, over 1m10s
31 runs so far, 0 failures, over 1m15s
65 runs so far, 0 failures, over 1m20s
85 runs so far, 0 failures, over 1m25s
95 runs so far, 0 failures, over 1m30s
99 runs so far, 0 failures, over 1m35s
100 runs so far, 0 failures, over 1m40s
100 runs so far, 0 failures, over 1m45s
100 runs so far, 0 failures, over 1m50s
100 runs so far, 0 failures, over 1m55s
100 runs so far, 0 failures, over 2m0s
100 runs so far, 0 failures, over 2m5s
100 runs so far, 0 failures, over 2m10s
101 runs so far, 0 failures, over 2m15s
103 runs so far, 0 failures, over 2m20s
108 runs so far, 0 failures, over 2m25s
128 runs so far, 0 failures, over 2m30s
152 runs so far, 0 failures, over 2m35s
172 runs so far, 0 failures, over 2m40s
184 runs so far, 0 failures, over 2m45s
191 runs so far, 0 failures, over 2m50s
197 runs so far, 0 failures, over 2m55s
200 runs so far, 0 failures, over 3m0s
200 runs so far, 0 failures, over 3m5s
200 runs so far, 0 failures, over 3m10s
200 runs so far, 0 failures, over 3m15s
200 runs so far, 0 failures, over 3m20s
200 runs so far, 0 failures, over 3m25s
200 runs so far, 0 failures, over 3m30s
204 runs so far, 0 failures, over 3m35s
209 runs so far, 0 failures, over 3m40s
226 runs so far, 0 failures, over 3m45s
239 runs so far, 0 failures, over 3m50s
260 runs so far, 0 failures, over 3m55s
281 runs so far, 0 failures, over 4m0s
289 runs so far, 0 failures, over 4m5s
294 runs so far, 0 failures, over 4m10s
298 runs so far, 0 failures, over 4m15s
299 runs so far, 0 failures, over 4m20s
300 runs so far, 0 failures, over 4m25s
300 runs so far, 0 failures, over 4m30s
300 runs so far, 0 failures, over 4m35s
301 runs so far, 0 failures, over 4m40s
302 runs so far, 0 failures, over 4m45s
304 runs so far, 0 failures, over 4m50s
305 runs so far, 0 failures, over 4m55s
316 runs so far, 0 failures, over 5m0s
334 runs so far, 0 failures, over 5m5s
350 runs so far, 0 failures, over 5m10s
369 runs so far, 0 failures, over 5m15s
382 runs so far, 0 failures, over 5m20s
391 runs so far, 0 failures, over 5m25s
395 runs so far, 0 failures, over 5m30s
396 runs so far, 0 failures, over 5m35s
396 runs so far, 0 failures, over 5m40s
400 runs so far, 0 failures, over 5m45s
400 runs so far, 0 failures, over 5m50s
402 runs so far, 0 failures, over 5m55s
402 runs so far, 0 failures, over 6m0s
403 runs so far, 0 failures, over 6m5s
406 runs so far, 0 failures, over 6m10s
413 runs so far, 0 failures, over 6m15s
421 runs so far, 0 failures, over 6m20s
441 runs so far, 0 failures, over 6m25s
456 runs so far, 0 failures, over 6m30s
471 runs so far, 0 failures, over 6m35s
482 runs so far, 0 failures, over 6m40s
490 runs so far, 0 failures, over 6m45s
492 runs so far, 0 failures, over 6m50s
497 runs so far, 0 failures, over 6m55s
499 runs so far, 0 failures, over 7m0s
501 runs so far, 0 failures, over 7m5s
503 runs so far, 0 failures, over 7m10s
503 runs so far, 0 failures, over 7m15s
503 runs so far, 0 failures, over 7m20s
505 runs so far, 0 failures, over 7m25s
511 runs so far, 0 failures, over 7m30s
520 runs so far, 0 failures, over 7m35s
529 runs so far, 0 failures, over 7m40s
543 runs so far, 0 failures, over 7m45s
557 runs so far, 0 failures, over 7m50s
571 runs so far, 0 failures, over 7m55s
581 runs so far, 0 failures, over 8m0s
587 runs so far, 0 failures, over 8m5s
595 runs so far, 0 failures, over 8m10s
597 runs so far, 0 failures, over 8m15s
600 runs so far, 0 failures, over 8m20s
602 runs so far, 0 failures, over 8m25s
603 runs so far, 0 failures, over 8m30s
603 runs so far, 0 failures, over 8m35s
604 runs so far, 0 failures, over 8m40s
608 runs so far, 0 failures, over 8m45s
615 runs so far, 0 failures, over 8m50s
624 runs so far, 0 failures, over 8m55s
632 runs so far, 0 failures, over 9m0s
648 runs so far, 0 failures, over 9m5s
657 runs so far, 0 failures, over 9m10s
671 runs so far, 0 failures, over 9m15s
683 runs so far, 0 failures, over 9m20s
689 runs so far, 0 failures, over 9m25s
693 runs so far, 0 failures, over 9m30s
697 runs so far, 0 failures, over 9m35s
699 runs so far, 0 failures, over 9m40s
703 runs so far, 0 failures, over 9m45s
703 runs so far, 0 failures, over 9m50s
703 runs so far, 0 failures, over 9m55s
704 runs so far, 0 failures, over 10m0s
708 runs so far, 0 failures, over 10m5s
718 runs so far, 0 failures, over 10m10s
725 runs so far, 0 failures, over 10m15s
739 runs so far, 0 failures, over 10m20s
751 runs so far, 0 failures, over 10m25s
764 runs so far, 0 failures, over 10m30s
773 runs so far, 0 failures, over 10m35s
783 runs so far, 0 failures, over 10m40s
788 runs so far, 0 failures, over 10m45s
794 runs so far, 0 failures, over 10m50s
797 runs so far, 0 failures, over 10m55s
801 runs so far, 0 failures, over 11m0s
802 runs so far, 0 failures, over 11m5s
804 runs so far, 0 failures, over 11m10s
805 runs so far, 0 failures, over 11m15s
806 runs so far, 0 failures, over 11m20s
813 runs so far, 0 failures, over 11m25s
822 runs so far, 0 failures, over 11m30s
827 runs so far, 0 failures, over 11m35s
837 runs so far, 0 failures, over 11m40s
852 runs so far, 0 failures, over 11m45s
864 runs so far, 0 failures, over 11m50s
877 runs so far, 0 failures, over 11m55s
880 runs so far, 0 failures, over 12m0s
891 runs so far, 0 failures, over 12m5s
893 runs so far, 0 failures, over 12m10s
898 runs so far, 0 failures, over 12m15s
902 runs so far, 0 failures, over 12m20s
903 runs so far, 0 failures, over 12m25s
904 runs so far, 0 failures, over 12m30s
905 runs so far, 0 failures, over 12m35s
908 runs so far, 0 failures, over 12m40s
914 runs so far, 0 failures, over 12m45s
921 runs so far, 0 failures, over 12m50s
929 runs so far, 0 failures, over 12m55s
940 runs so far, 0 failures, over 13m0s
952 runs so far, 0 failures, over 13m5s
967 runs so far, 0 failures, over 13m10s
979 runs so far, 0 failures, over 13m15s
984 runs so far, 0 failures, over 13m20s
989 runs so far, 0 failures, over 13m25s
994 runs so far, 0 failures, over 13m30s
997 runs so far, 0 failures, over 13m35s
1002 runs so far, 0 failures, over 13m40s
1005 runs so far, 0 failures, over 13m45s
1006 runs so far, 0 failures, over 13m50s
1008 runs so far, 0 failures, over 13m55s
1011 runs so far, 0 failures, over 14m0s
1016 runs so far, 0 failures, over 14m5s
1022 runs so far, 0 failures, over 14m10s
1030 runs so far, 0 failures, over 14m15s
1036 runs so far, 0 failures, over 14m20s
1051 runs so far, 0 failures, over 14m25s
1069 runs so far, 0 failures, over 14m30s
1083 runs so far, 0 failures, over 14m35s
1087 runs so far, 0 failures, over 14m40s
1092 runs so far, 0 failures, over 14m45s
1095 runs so far, 0 failures, over 14m50s
1098 runs so far, 0 failures, over 14m55s
1102 runs so far, 0 failures, over 15m0s
1105 runs so far, 0 failures, over 15m5s
1106 runs so far, 0 failures, over 15m10s
1109 runs so far, 0 failures, over 15m15s
1111 runs so far, 0 failures, over 15m20s
1117 runs so far, 0 failures, over 15m25s
1122 runs so far, 0 failures, over 15m30s
1131 runs so far, 0 failures, over 15m35s
1140 runs so far, 0 failures, over 15m40s
1153 runs so far, 0 failures, over 15m45s
1165 runs so far, 0 failures, over 15m50s
1179 runs so far, 0 failures, over 15m55s
1188 runs so far, 0 failures, over 16m0s
1192 runs so far, 0 failures, over 16m5s
1199 runs so far, 0 failures, over 16m10s
1202 runs so far, 0 failures, over 16m15s
1203 runs so far, 0 failures, over 16m20s
1205 runs so far, 0 failures, over 16m25s
1208 runs so far, 0 failures, over 16m30s
1209 runs so far, 0 failures, over 16m35s

I161111 17:23:08.530705 1 rand.go:76  Random seed: -8922889892323237005
I161111 17:23:08.895875 25 storage/engine/rocksdb.go:340  opening in memory rocksdb instance
I161111 17:23:08.897883 25 gossip/gossip.go:237  [n?] initial resolvers: []
W161111 17:23:08.897913 25 gossip/gossip.go:1055  [n?] no resolvers found; use --join to specify a connected node
I161111 17:23:08.897981 25 base/node_id.go:62  NodeID set to 1
I161111 17:23:08.902511 25 storage/store.go:1187  [n1,s1]: failed initial metrics computation: [n1,s1]: system config not yet available
I161111 17:23:08.902674 25 gossip/gossip.go:280  [n1] NodeDescriptor set to node_id:1 address:<network_field:"tcp" address_field:"127.0.0.1:46606" > attrs:<> locality:<>
I161111 17:23:08.903917 124 raft/raft.go:568  [s1,r1/1:/M{in-ax}] 1 became follower at term 5
I161111 17:23:08.904074 124 raft/raft.go:316  [s1,r1/1:/M{in-ax}] newRaft 1 [peers: [1], term: 5, commit: 10, applied: 10, lastindex: 10, lastterm: 5]
I161111 17:23:08.904135 124 raft/raft.go:750  [s1,r1/1:/M{in-ax}] 1 is starting a new election at term 5
I161111 17:23:08.904257 124 raft/raft.go:581  [s1,r1/1:/M{in-ax}] 1 became candidate at term 6
I161111 17:23:08.904309 124 raft/raft.go:665  [s1,r1/1:/M{in-ax}] 1 received MsgVoteResp from 1 at term 6
I161111 17:23:08.904347 124 raft/raft.go:622  [s1,r1/1:/M{in-ax}] 1 became leader at term 6
I161111 17:23:08.905360 16 storage/replica_proposal.go:328  [s1,r1/1:/M{in-ax}] new range lease replica {1 1 1} 1970-01-01 00:00:00 +0000 UTC 900.000124ms following replica {0 0 0} 1970-01-01 00:00:00 +0000 UTC 0s [physicalTime=1970-01-01 00:00:00.000000123 +0000 UTC]
I161111 17:23:08.906471 25 storage/engine/rocksdb.go:340  opening in memory rocksdb instance
I161111 17:23:08.907451 25 gossip/gossip.go:237  [n?] initial resolvers: [127.0.0.1:46606]
W161111 17:23:08.907499 25 gossip/gossip.go:1057  [n?] no incoming or outgoing connections
I161111 17:23:08.907556 25 base/node_id.go:62  NodeID set to 2
I161111 17:23:08.934476 25 storage/store.go:1187  [n2,s2]: failed initial metrics computation: [n2,s2]: system config not yet available
I161111 17:23:08.934534 25 gossip/gossip.go:280  [n2] NodeDescriptor set to node_id:2 address:<network_field:"tcp" address_field:"127.0.0.1:42236" > attrs:<> locality:<>
I161111 17:23:08.935222 25 storage/engine/rocksdb.go:340  opening in memory rocksdb instance
I161111 17:23:08.936252 25 gossip/gossip.go:237  [n?] initial resolvers: [127.0.0.1:46606]
W161111 17:23:08.936276 25 gossip/gossip.go:1057  [n?] no incoming or outgoing connections
I161111 17:23:08.936329 25 base/node_id.go:62  NodeID set to 3
I161111 17:23:08.938793 25 storage/store.go:1187  [n3,s3]: failed initial metrics computation: [n3,s3]: system config not yet available
I161111 17:23:08.938849 25 gossip/gossip.go:280  [n3] NodeDescriptor set to node_id:3 address:<network_field:"tcp" address_field:"127.0.0.1:42944" > attrs:<> locality:<>
I161111 17:23:09.097817 307 gossip/client.go:125  [n3] started gossip client to 127.0.0.1:46606
I161111 17:23:09.098030 58 gossip/client.go:125  [n2] started gossip client to 127.0.0.1:46606
I161111 17:23:09.108615 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot e20d01e1 at index 13 in 52.652µs.
I161111 17:23:09.109642 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 32, log entries: 3
I161111 17:23:09.109932 356 storage/store.go:1912  [s2] ADD 1
I161111 17:23:09.109990 356 raft/raft.go:568  [s2,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.110096 356 raft/raft.go:316  [s2,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.110118 356 raft/raft.go:568  [s2,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.110148 356 raft/raft.go:710  [s2,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.110160 356 raft/raft.go:568  [s2,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.110207 356 raft/raft.go:1125  [s2,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 13, term: 6]
I161111 17:23:09.110234 356 raft/log.go:288  [s2,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 13, term: 6]
I161111 17:23:09.110267 356 raft/raft.go:1135  [s2,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 14, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.110289 356 raft/raft.go:1102  [s2,r1/?:{-}] ffffffffffffffff [commit: 13] restored snapshot [index: 13, term: 6]
I161111 17:23:09.110317 356 storage/replica_raftstorage.go:589  [s2,r1/?:{-}] applying preemptive snapshot at index 13 (id=e20d01e1, encoded size=16, 1 rocksdb batches, 3 log entries)
I161111 17:23:09.110762 356 storage/replica_raftstorage.go:592  [s2,r1/?:/M{in-ax}] applied preemptive snapshot in 0.000s
I161111 17:23:09.110796 356 storage/store.go:2817  [s2,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.111345 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > next_replica_id:2
I161111 17:23:09.143093 372 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:2 StoreID:2 ReplicaID:2}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:09.145635 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 4eab6763 at index 19 in 47.682µs.
I161111 17:23:09.156412 321 storage/store.go:1912  [s3] ADD 1
I161111 17:23:09.156487 321 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.156574 321 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.156590 321 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.156611 321 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.156622 321 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.156713 321 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 19, term: 6]
I161111 17:23:09.156735 321 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 19, term: 6]
I161111 17:23:09.156753 321 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 20, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.156767 321 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 20, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.156802 321 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 19] restored snapshot [index: 19, term: 6]
I161111 17:23:09.156829 321 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 19 (id=4eab6763, encoded size=16, 1 rocksdb batches, 9 log entries)
I161111 17:23:09.157673 321 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:09.157718 321 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.158285 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 39, log entries: 9
I161111 17:23:09.158819 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:3
I161111 17:23:09.165061 188 raft/raft.go:568  [s2,r1/?:/M{in-ax}] 2 became follower at term 6
I161111 17:23:09.165188 188 raft/raft.go:316  [s2,r1/?:/M{in-ax}] newRaft 2 [peers: [1], term: 6, commit: 13, applied: 13, lastindex: 13, lastterm: 6]
I161111 17:23:09.165231 188 raft/raft.go:1088  [s2,r1/?:/M{in-ax}] 2 [logterm: 0, index: 19] rejected msgApp [logterm: 6, index: 19] from 1
I161111 17:23:09.165470 389 storage/raft_transport.go:432  raft transport stream to node 1 established
I161111 17:23:09.165764 73 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 13) from 2 for index 19
I161111 17:23:09.165810 73 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2 to [next = 14, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.168519 395 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:3}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:3}]
I161111 17:23:09.263039 241 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3 became follower at term 6
I161111 17:23:09.263150 241 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3 [peers: [1,2], term: 6, commit: 19, applied: 19, lastindex: 19, lastterm: 6]
I161111 17:23:09.308645 450 storage/raft_transport.go:432  raft transport stream to node 1 established
I161111 17:23:09.317085 244 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3 [logterm: 0, index: 21] rejected msgApp [logterm: 6, index: 21] from 1
I161111 17:23:09.317151 244 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3 [logterm: 0, index: 21] rejected msgApp [logterm: 6, index: 21] from 1
I161111 17:23:09.317478 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 19) from 3 for index 21
I161111 17:23:09.317536 91 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3 to [next = 20, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.317620 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 19) from 3 for index 21
I161111 17:23:09.429322 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:3 > next_replica_id:4
I161111 17:23:09.431579 417 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:3}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:09.434189 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 81c1126b at index 26 in 49.41µs.
I161111 17:23:09.439593 340 storage/store.go:2041  [s3,r1/3:/M{in-ax}] DEL 1
I161111 17:23:09.440328 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 48, log entries: 16
W161111 17:23:09.456669 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 3}: raft group deleted
I161111 17:23:09.605315 499 storage/store.go:1912  [s3] ADD 1
I161111 17:23:09.605436 499 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.605620 499 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.605672 499 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.605704 499 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.605749 499 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.605844 499 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 26, term: 6]
I161111 17:23:09.605897 499 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 26, term: 6]
I161111 17:23:09.605928 499 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 27, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.605953 499 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 27, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.606010 499 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 26] restored snapshot [index: 26, term: 6]
I161111 17:23:09.606039 499 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 26 (id=81c1126b, encoded size=16, 1 rocksdb batches, 16 log entries)
I161111 17:23:09.607282 499 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:09.607334 499 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.608066 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:4
I161111 17:23:09.609600 473 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:4}]
I161111 17:23:09.638588 256 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4 became follower at term 6
I161111 17:23:09.638631 256 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4 [peers: [1,2], term: 6, commit: 26, applied: 26, lastindex: 26, lastterm: 6]
I161111 17:23:09.638678 256 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4 [logterm: 0, index: 31] rejected msgApp [logterm: 6, index: 31] from 1
I161111 17:23:09.638873 86 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 26) from 4 for index 31
I161111 17:23:09.638899 86 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4 to [next = 27, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.664496 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:4 > next_replica_id:5
I161111 17:23:09.666986 549 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:4}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:09.669762 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 49471085 at index 35 in 63.133µs.
I161111 17:23:09.671983 340 storage/store.go:2041  [s3,r1/4:/M{in-ax}] DEL 1
I161111 17:23:09.673057 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 59, log entries: 25
I161111 17:23:09.676689 585 storage/store.go:1912  [s3] ADD 1
I161111 17:23:09.676779 585 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.676912 585 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.676944 585 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.677210 585 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.677238 585 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.677312 585 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 35, term: 6]
I161111 17:23:09.677348 585 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 35, term: 6]
I161111 17:23:09.677374 585 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 36, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.677401 585 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 36, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.677422 585 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 35] restored snapshot [index: 35, term: 6]
I161111 17:23:09.677446 585 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 35 (id=49471085, encoded size=16, 1 rocksdb batches, 25 log entries)
I161111 17:23:09.678440 585 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:09.678477 585 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.679138 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:5
I161111 17:23:09.681323 589 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:5}]
I161111 17:23:09.707511 273 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5 became follower at term 6
I161111 17:23:09.707557 273 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5 [peers: [1,2], term: 6, commit: 35, applied: 35, lastindex: 35, lastterm: 6]
I161111 17:23:09.737467 275 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5 [logterm: 0, index: 38] rejected msgApp [logterm: 6, index: 38] from 1
I161111 17:23:09.737701 87 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 35) from 5 for index 38
I161111 17:23:09.737740 87 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5 to [next = 36, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.797449 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:5 > next_replica_id:6
I161111 17:23:09.799089 496 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:5}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:09.802057 340 storage/store.go:2041  [s3,r1/5:/M{in-ax}] DEL 1
I161111 17:23:09.824950 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot cf26998c at index 40 in 52.299µs.
I161111 17:23:09.837553 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 66, log entries: 30
I161111 17:23:09.839034 594 storage/store.go:1912  [s3] ADD 1
I161111 17:23:09.839109 594 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.839251 594 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.839286 594 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.839324 594 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.839350 594 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.839435 594 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 40, term: 6]
I161111 17:23:09.839464 594 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 40, term: 6]
I161111 17:23:09.839495 594 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 41, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.839523 594 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 41, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.839545 594 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 40] restored snapshot [index: 40, term: 6]
I161111 17:23:09.839573 594 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 40 (id=cf26998c, encoded size=16, 1 rocksdb batches, 30 log entries)
I161111 17:23:09.840666 594 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:09.840729 594 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.860801 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:6
I161111 17:23:09.864040 504 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:6}]
I161111 17:23:09.906111 243 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6 became follower at term 6
I161111 17:23:09.906253 243 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6 [peers: [1,2], term: 6, commit: 40, applied: 40, lastindex: 40, lastterm: 6]
I161111 17:23:09.942287 245 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6 [logterm: 0, index: 45] rejected msgApp [logterm: 6, index: 45] from 1
I161111 17:23:09.947549 81 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 40) from 6 for index 45
I161111 17:23:09.952764 81 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6 to [next = 41, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.969041 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:6 > next_replica_id:7
I161111 17:23:09.971028 601 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:6}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:09.973553 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 734bccb2 at index 47 in 64.551µs.
I161111 17:23:09.974311 452 storage/store.go:2988  [s3,r1/6:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:09.975249 340 storage/store.go:2041  [s3,r1/6:/M{in-ax}] DEL 1
I161111 17:23:09.976174 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 75, log entries: 37
W161111 17:23:09.976210 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 6}: raft group deleted
I161111 17:23:09.976497 644 storage/store.go:1912  [s3] ADD 1
I161111 17:23:09.976554 644 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:09.976671 644 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:09.976689 644 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:09.976712 644 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:09.976724 644 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:09.976803 644 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 47, term: 6]
I161111 17:23:09.976822 644 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 47, term: 6]
I161111 17:23:09.976840 644 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 48, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.976855 644 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 48, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:09.976867 644 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 47] restored snapshot [index: 47, term: 6]
I161111 17:23:09.976885 644 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 47 (id=734bccb2, encoded size=16, 1 rocksdb batches, 37 log entries)
I161111 17:23:09.978115 644 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:09.978133 644 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:09.978890 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:7
I161111 17:23:09.981286 608 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:7}]
I161111 17:23:10.004212 257 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7 became follower at term 6
I161111 17:23:10.004282 257 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7 [peers: [1,2], term: 6, commit: 47, applied: 47, lastindex: 47, lastterm: 6]
I161111 17:23:10.004368 257 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7 [logterm: 0, index: 49] rejected msgApp [logterm: 6, index: 49] from 1
I161111 17:23:10.004595 105 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 47) from 7 for index 49
I161111 17:23:10.004622 105 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7 to [next = 48, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.006691 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:7 > next_replica_id:8
I161111 17:23:10.009020 527 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:7}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.052873 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 76d2cdc5 at index 53 in 79.304µs.
I161111 17:23:10.116802 340 storage/store.go:2041  [s3,r1/7:/M{in-ax}] DEL 1
I161111 17:23:10.117193 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 83, log entries: 43
I161111 17:23:10.118019 438 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.118074 438 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.118232 438 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.118248 438 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.118270 438 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.118282 438 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.118414 438 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 53, term: 6]
I161111 17:23:10.118430 438 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 53, term: 6]
I161111 17:23:10.118450 438 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 54, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.118470 438 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 54, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.118495 438 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 53] restored snapshot [index: 53, term: 6]
I161111 17:23:10.118515 438 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 53 (id=76d2cdc5, encoded size=16, 1 rocksdb batches, 43 log entries)
I161111 17:23:10.119927 438 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.001s
I161111 17:23:10.119943 438 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.120278 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.121282 340 storage/store.go:2041  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.122237 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:8
I161111 17:23:10.129854 707 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:8}]
I161111 17:23:10.182030 265 raft/raft.go:568  [s3,r1/8:{-}] 8 became follower at term 0
I161111 17:23:10.182268 265 raft/raft.go:316  [s3,r1/8:{-}] newRaft 8 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.182287 265 raft/raft.go:568  [s3,r1/8:{-}] 8 became follower at term 1
I161111 17:23:10.182311 265 raft/raft.go:710  [s3,r1/8:{-}] 8 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:10.182323 265 raft/raft.go:568  [s3,r1/8:{-}] 8 became follower at term 6
I161111 17:23:10.182342 265 raft/raft.go:1088  [s3,r1/8:{-}] 8 [logterm: 0, index: 56] rejected msgApp [logterm: 6, index: 56] from 1
I161111 17:23:10.182570 84 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 8 for index 56
I161111 17:23:10.182599 84 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 8 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.182717 84 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a786452f at index 56 in 70.186µs.
I161111 17:23:10.182790 84 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 56] sent snapshot[index: 56, term: 6] to 8 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.182822 84 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 8 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 56]
I161111 17:23:10.188181 512 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 87, log entries: 46
I161111 17:23:10.188431 446 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.189007 446 raft/raft.go:1125  [s3,r1/8:{-}] 8 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 56, term: 6]
I161111 17:23:10.189114 446 raft/log.go:288  [s3,r1/8:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 56, term: 6]
I161111 17:23:10.189323 446 raft/raft.go:1135  [s3,r1/8:{-}] 8 restored progress of 1 [next = 57, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.189353 446 raft/raft.go:1135  [s3,r1/8:{-}] 8 restored progress of 2 [next = 57, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.189471 446 raft/raft.go:1135  [s3,r1/8:{-}] 8 restored progress of 8 [next = 57, match = 56, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.189568 446 raft/raft.go:1102  [s3,r1/8:{-}] 8 [commit: 56] restored snapshot [index: 56, term: 6]
I161111 17:23:10.189659 446 storage/replica_raftstorage.go:589  [s3,r1/8:{-}] applying Raft snapshot at index 56 (id=a786452f, encoded size=16, 1 rocksdb batches, 46 log entries)
I161111 17:23:10.193337 446 storage/replica_raftstorage.go:592  [s3,r1/8:/M{in-ax}] applied Raft snapshot in 0.004s
I161111 17:23:10.193837 92 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 8 [next = 57, match = 56, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 56]
I161111 17:23:10.217263 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:8 > next_replica_id:9
I161111 17:23:10.219081 742 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:8}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.221114 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0c34daa0 at index 61 in 53.969µs.
I161111 17:23:10.221556 340 storage/store.go:2041  [s3,r1/8:/M{in-ax}] DEL 1
I161111 17:23:10.222401 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 93, log entries: 51
I161111 17:23:10.222620 749 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.222671 749 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.222860 749 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.222875 749 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.222895 749 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.222907 749 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.223060 749 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 61, term: 6]
I161111 17:23:10.223076 749 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 61, term: 6]
I161111 17:23:10.223096 749 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 62, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.223111 749 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 62, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.223123 749 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 61] restored snapshot [index: 61, term: 6]
I161111 17:23:10.223140 749 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 61 (id=0c34daa0, encoded size=16, 1 rocksdb batches, 51 log entries)
I161111 17:23:10.224918 749 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161111 17:23:10.224961 749 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.233518 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:9
I161111 17:23:10.235693 652 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:9}]
I161111 17:23:10.267440 240 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 8 became follower at term 6
I161111 17:23:10.267515 240 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 8 [peers: [1,2], term: 6, commit: 61, applied: 61, lastindex: 61, lastterm: 6]
I161111 17:23:10.290455 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.355925 243 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 9 became follower at term 6
I161111 17:23:10.355979 243 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 9 [peers: [1,2], term: 6, commit: 61, applied: 61, lastindex: 61, lastterm: 6]
I161111 17:23:10.359146 247 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 9 [logterm: 0, index: 63] rejected msgApp [logterm: 6, index: 63] from 1
I161111 17:23:10.359306 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 61) from 9 for index 63
I161111 17:23:10.359336 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 9 to [next = 62, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.426984 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:9 > next_replica_id:10
I161111 17:23:10.429749 687 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:9}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.431970 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 6d4bef4b at index 68 in 50.325µs.
I161111 17:23:10.438972 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 102, log entries: 58
I161111 17:23:10.439301 452 storage/store.go:2988  [s3,r1/9:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.440021 340 storage/store.go:2041  [s3,r1/9:/M{in-ax}] DEL 1
I161111 17:23:10.440291 689 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.440367 689 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.440608 689 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.440655 689 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.440745 689 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.440772 689 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.440952 689 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 68, term: 6]
I161111 17:23:10.440985 689 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 68, term: 6]
I161111 17:23:10.441016 689 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 69, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.441047 689 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 69, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.441073 689 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 68] restored snapshot [index: 68, term: 6]
I161111 17:23:10.441102 689 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 68 (id=6d4bef4b, encoded size=16, 1 rocksdb batches, 58 log entries)
I161111 17:23:10.443043 689 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161111 17:23:10.451312 689 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.456306 258 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 9 became follower at term 6
I161111 17:23:10.456356 258 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 9 [peers: [1,2], term: 6, commit: 68, applied: 68, lastindex: 68, lastterm: 6]
I161111 17:23:10.481968 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:10
I161111 17:23:10.483516 786 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:10}]
I161111 17:23:10.493567 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.538006 263 raft/raft.go:568  [s3,r1/?:/M{in-ax}] a became follower at term 6
I161111 17:23:10.538047 263 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft a [peers: [1,2], term: 6, commit: 68, applied: 68, lastindex: 68, lastterm: 6]
I161111 17:23:10.538070 263 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] a [logterm: 0, index: 70] rejected msgApp [logterm: 6, index: 70] from 1
I161111 17:23:10.538225 94 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 68) from a for index 70
I161111 17:23:10.538252 94 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of a to [next = 69, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.569322 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:10 > next_replica_id:11
I161111 17:23:10.571047 836 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:10}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.573389 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9c731cd7 at index 74 in 86.931µs.
I161111 17:23:10.632378 452 storage/store.go:2988  [s3,r1/10:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.634895 340 storage/store.go:2041  [s3,r1/10:/M{in-ax}] DEL 1
I161111 17:23:10.638821 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 110, log entries: 64
I161111 17:23:10.640349 760 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.640435 760 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.640788 760 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.640815 760 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.640848 760 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.640867 760 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.641083 760 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 74, term: 6]
I161111 17:23:10.641114 760 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 74, term: 6]
I161111 17:23:10.641145 760 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 75, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.641170 760 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 75, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.641189 760 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 74] restored snapshot [index: 74, term: 6]
I161111 17:23:10.641232 760 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 74 (id=9c731cd7, encoded size=16, 1 rocksdb batches, 64 log entries)
I161111 17:23:10.691373 760 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.050s
I161111 17:23:10.691405 760 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.692821 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:11
I161111 17:23:10.698716 870 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:11}]
I161111 17:23:10.704721 283 raft/raft.go:568  [s3,r1/?:/M{in-ax}] b became follower at term 6
I161111 17:23:10.704764 283 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft b [peers: [1,2], term: 6, commit: 74, applied: 74, lastindex: 74, lastterm: 6]
I161111 17:23:10.723254 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] b [logterm: 0, index: 77] rejected msgApp [logterm: 6, index: 77] from 1
I161111 17:23:10.723437 93 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 74) from b for index 77
I161111 17:23:10.723473 93 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of b to [next = 75, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.742937 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:11 > next_replica_id:12
I161111 17:23:10.744911 872 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:11}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.746883 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7f371ad9 at index 82 in 51.758µs.
I161111 17:23:10.757349 452 storage/store.go:2988  [s3,r1/11:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.758719 340 storage/store.go:2041  [s3,r1/11:/M{in-ax}] DEL 1
I161111 17:23:10.780131 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 120, log entries: 72
I161111 17:23:10.782887 838 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.791841 838 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.792320 838 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.792342 838 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.792389 838 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.792405 838 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.792689 838 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 82, term: 6]
I161111 17:23:10.792732 838 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 82, term: 6]
I161111 17:23:10.792754 838 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 83, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.792772 838 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 83, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.792819 838 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 82] restored snapshot [index: 82, term: 6]
I161111 17:23:10.792842 838 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 82 (id=7f371ad9, encoded size=16, 1 rocksdb batches, 72 log entries)
I161111 17:23:10.839628 838 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.047s
I161111 17:23:10.839660 838 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.840400 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:12
I161111 17:23:10.842726 902 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:12}]
I161111 17:23:10.898149 253 raft/raft.go:568  [s3,r1/?:/M{in-ax}] c became follower at term 6
I161111 17:23:10.898191 253 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft c [peers: [1,2], term: 6, commit: 82, applied: 82, lastindex: 82, lastterm: 6]
I161111 17:23:10.898232 253 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] c [logterm: 0, index: 84] rejected msgApp [logterm: 6, index: 84] from 1
I161111 17:23:10.899250 96 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 82) from c for index 84
I161111 17:23:10.899273 96 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of c to [next = 83, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.908637 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:12 > next_replica_id:13
I161111 17:23:10.910430 893 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:12}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:10.912809 452 storage/store.go:2988  [s3,r1/12:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:10.914381 340 storage/store.go:2041  [s3,r1/12:/M{in-ax}] DEL 1
W161111 17:23:10.915354 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 12}: raft group deleted
I161111 17:23:10.936273 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 5d1e787d at index 89 in 145.553µs.
W161111 17:23:10.954018 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 12}: raft group deleted
I161111 17:23:10.956845 848 storage/store.go:1912  [s3] ADD 1
I161111 17:23:10.956930 848 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:10.957301 848 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:10.957323 848 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:10.957351 848 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:10.957367 848 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:10.957606 848 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 89, term: 6]
I161111 17:23:10.957631 848 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 89, term: 6]
I161111 17:23:10.957661 848 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 90, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.957683 848 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 90, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:10.957700 848 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 89] restored snapshot [index: 89, term: 6]
I161111 17:23:10.957725 848 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 89 (id=5d1e787d, encoded size=16, 1 rocksdb batches, 79 log entries)
I161111 17:23:10.969703 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 129, log entries: 79
I161111 17:23:10.978010 848 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.020s
I161111 17:23:10.978075 848 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:10.978870 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:13
I161111 17:23:10.980474 932 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:13}]
I161111 17:23:11.054359 266 raft/raft.go:568  [s3,r1/?:/M{in-ax}] d became follower at term 6
I161111 17:23:11.054476 266 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft d [peers: [1,2], term: 6, commit: 89, applied: 89, lastindex: 89, lastterm: 6]
I161111 17:23:11.080552 268 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] d [logterm: 0, index: 91] rejected msgApp [logterm: 6, index: 91] from 1
I161111 17:23:11.080737 87 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 89) from d for index 91
I161111 17:23:11.080763 87 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of d to [next = 90, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.146134 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:13 > next_replica_id:14
I161111 17:23:11.148047 794 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:13}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:11.150434 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot bdf0692b at index 96 in 49.373µs.
I161111 17:23:11.160935 452 storage/store.go:2988  [s3,r1/13:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:11.163228 340 storage/store.go:2041  [s3,r1/13:/M{in-ax}] DEL 1
W161111 17:23:11.199672 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 13}: raft group deleted
I161111 17:23:11.199717 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 138, log entries: 86
I161111 17:23:11.199959 954 storage/store.go:1912  [s3] ADD 1
I161111 17:23:11.200010 954 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:11.200406 954 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:11.200424 954 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:11.200445 954 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:11.200457 954 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:11.200701 954 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 96, term: 6]
I161111 17:23:11.200721 954 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 96, term: 6]
I161111 17:23:11.200742 954 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 97, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.200758 954 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 97, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.200771 954 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 96] restored snapshot [index: 96, term: 6]
I161111 17:23:11.200792 954 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 96 (id=bdf0692b, encoded size=16, 1 rocksdb batches, 86 log entries)
I161111 17:23:11.203915 954 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161111 17:23:11.203946 954 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:11.204699 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:14
I161111 17:23:11.206601 935 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:14}]
I161111 17:23:11.266253 242 raft/raft.go:568  [s3,r1/?:/M{in-ax}] e became follower at term 6
I161111 17:23:11.266298 242 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft e [peers: [1,2], term: 6, commit: 96, applied: 96, lastindex: 96, lastterm: 6]
I161111 17:23:11.266341 242 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] e [logterm: 0, index: 98] rejected msgApp [logterm: 6, index: 98] from 1
I161111 17:23:11.266560 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 96) from e for index 98
I161111 17:23:11.266585 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of e to [next = 97, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.304774 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:14 > next_replica_id:15
I161111 17:23:11.317337 1012 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:14}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:11.321621 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot fffab62a at index 103 in 83.936µs.
I161111 17:23:11.416535 340 storage/store.go:2041  [s3,r1/14:/M{in-ax}] DEL 1
W161111 17:23:11.423462 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 14}: raft group deleted
I161111 17:23:11.452550 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 147, log entries: 93
I161111 17:23:11.452864 955 storage/store.go:1912  [s3] ADD 1
I161111 17:23:11.452922 955 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:11.453343 955 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:11.453369 955 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:11.453396 955 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:11.453407 955 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:11.453668 955 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 103, term: 6]
I161111 17:23:11.453687 955 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 103, term: 6]
I161111 17:23:11.453732 955 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 104, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.453753 955 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 104, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.453766 955 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 103] restored snapshot [index: 103, term: 6]
I161111 17:23:11.453783 955 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 103 (id=fffab62a, encoded size=16, 1 rocksdb batches, 93 log entries)
I161111 17:23:11.461958 955 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.008s
I161111 17:23:11.461982 955 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:11.467869 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:15
I161111 17:23:11.469545 1058 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:15}]
I161111 17:23:11.474829 265 raft/raft.go:568  [s3,r1/?:/M{in-ax}] f became follower at term 6
I161111 17:23:11.474898 265 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft f [peers: [1,2], term: 6, commit: 103, applied: 103, lastindex: 103, lastterm: 6]
I161111 17:23:11.474954 265 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] f [logterm: 0, index: 107] rejected msgApp [logterm: 6, index: 107] from 1
I161111 17:23:11.475139 104 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 103) from f for index 107
I161111 17:23:11.475179 104 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of f to [next = 104, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.481151 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:15 > next_replica_id:16
I161111 17:23:11.495684 639 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:15}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:11.497764 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot cdad5a48 at index 110 in 52.895µs.
I161111 17:23:11.499003 452 storage/store.go:2988  [s3,r1/15:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:11.499541 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 156, log entries: 100
I161111 17:23:11.501375 340 storage/store.go:2041  [s3,r1/15:/M{in-ax}] DEL 1
I161111 17:23:11.501749 1067 storage/store.go:1912  [s3] ADD 1
I161111 17:23:11.501828 1067 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:11.502256 1067 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:11.502274 1067 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:11.502316 1067 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:11.502331 1067 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:11.502681 1067 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 110, term: 6]
I161111 17:23:11.502726 1067 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 110, term: 6]
I161111 17:23:11.502750 1067 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 111, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.502765 1067 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 111, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.502796 1067 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 110] restored snapshot [index: 110, term: 6]
I161111 17:23:11.502816 1067 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 110 (id=cdad5a48, encoded size=16, 1 rocksdb batches, 100 log entries)
I161111 17:23:11.562577 1067 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.060s
I161111 17:23:11.562613 1067 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:11.630373 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:16
I161111 17:23:11.707428 1070 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:16}]
I161111 17:23:11.798735 283 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 10 became follower at term 6
I161111 17:23:11.798779 283 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 10 [peers: [1,2], term: 6, commit: 110, applied: 110, lastindex: 110, lastterm: 6]
I161111 17:23:11.798838 283 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 10 [logterm: 0, index: 115] rejected msgApp [logterm: 6, index: 115] from 1
I161111 17:23:11.799025 66 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 110) from 10 for index 115
I161111 17:23:11.799060 66 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 10 to [next = 111, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.827644 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:16 > next_replica_id:17
I161111 17:23:11.830778 1097 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:16}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:11.833341 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot aa3496e8 at index 119 in 52.627µs.
I161111 17:23:11.842569 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 167, log entries: 109
I161111 17:23:11.843194 452 storage/store.go:2988  [s3,r1/16:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:11.849725 340 storage/store.go:2041  [s3,r1/16:/M{in-ax}] DEL 1
I161111 17:23:11.849968 984 storage/store.go:1912  [s3] ADD 1
I161111 17:23:11.850020 984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:11.850603 984 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:11.850639 984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:11.850661 984 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:11.850673 984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:11.851070 984 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 119, term: 6]
I161111 17:23:11.851090 984 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 119, term: 6]
I161111 17:23:11.851129 984 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 120, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.851146 984 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 120, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:11.851162 984 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 119] restored snapshot [index: 119, term: 6]
I161111 17:23:11.851194 984 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 119 (id=aa3496e8, encoded size=16, 1 rocksdb batches, 109 log entries)
I161111 17:23:11.916747 984 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.066s
I161111 17:23:11.916788 984 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:11.917603 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:17
I161111 17:23:11.919801 1039 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:17}]
I161111 17:23:11.978463 248 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 11 became follower at term 6
I161111 17:23:11.978503 248 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 11 [peers: [1,2], term: 6, commit: 119, applied: 119, lastindex: 119, lastterm: 6]
I161111 17:23:11.978554 248 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 11 [logterm: 0, index: 122] rejected msgApp [logterm: 6, index: 122] from 1
I161111 17:23:11.978712 106 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 119) from 11 for index 122
I161111 17:23:11.978733 106 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 11 to [next = 120, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.008038 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:17 > next_replica_id:18
I161111 17:23:12.014288 1157 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:17}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.021037 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot b77f70b6 at index 127 in 81.242µs.
I161111 17:23:12.031875 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 177, log entries: 117
I161111 17:23:12.039325 1171 raft/raft.go:568  [s3,r1/17:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:12.046128 1171 raft/raft.go:316  [s3,r1/17:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 127, applied: 127, lastindex: 127, lastterm: 6]
I161111 17:23:12.046228 1171 raft/raft.go:1106  [s3,r1/17:/M{in-ax}] ffffffffffffffff [commit: 127] ignored snapshot [index: 127, term: 6]
I161111 17:23:12.048104 340 storage/store.go:2041  [s3,r1/17:/M{in-ax}] DEL 1
I161111 17:23:12.051359 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:18
I161111 17:23:12.065735 1135 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:18}]
I161111 17:23:12.104040 267 raft/raft.go:568  [s3,r1/18:{-}] 12 became follower at term 0
I161111 17:23:12.104579 267 raft/raft.go:316  [s3,r1/18:{-}] newRaft 12 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.104596 267 raft/raft.go:568  [s3,r1/18:{-}] 12 became follower at term 1
I161111 17:23:12.104617 267 raft/raft.go:710  [s3,r1/18:{-}] 12 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:12.104629 267 raft/raft.go:568  [s3,r1/18:{-}] 12 became follower at term 6
I161111 17:23:12.137434 269 raft/raft.go:1088  [s3,r1/18:{-}] 12 [logterm: 0, index: 129] rejected msgApp [logterm: 6, index: 129] from 1
I161111 17:23:12.137588 112 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 12 for index 129
I161111 17:23:12.137615 112 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 12 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.137749 112 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 43fbe028 at index 129 in 86.027µs.
I161111 17:23:12.137783 112 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 11, commit: 129] sent snapshot[index: 129, term: 6] to 12 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.137799 112 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 12 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 129]
I161111 17:23:12.139255 1198 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 180, log entries: 119
I161111 17:23:12.139385 1201 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.139945 1201 raft/raft.go:1125  [s3,r1/18:{-}] 12 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 129, term: 6]
I161111 17:23:12.139963 1201 raft/log.go:288  [s3,r1/18:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 129, term: 6]
I161111 17:23:12.139983 1201 raft/raft.go:1135  [s3,r1/18:{-}] 12 restored progress of 1 [next = 130, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.140014 1201 raft/raft.go:1135  [s3,r1/18:{-}] 12 restored progress of 2 [next = 130, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.140031 1201 raft/raft.go:1135  [s3,r1/18:{-}] 12 restored progress of 12 [next = 130, match = 129, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.140043 1201 raft/raft.go:1102  [s3,r1/18:{-}] 12 [commit: 129] restored snapshot [index: 129, term: 6]
I161111 17:23:12.140068 1201 storage/replica_raftstorage.go:589  [s3,r1/18:{-}] applying Raft snapshot at index 129 (id=43fbe028, encoded size=16, 1 rocksdb batches, 119 log entries)
I161111 17:23:12.143978 1201 storage/replica_raftstorage.go:592  [s3,r1/18:/M{in-ax}] applied Raft snapshot in 0.004s
I161111 17:23:12.144398 16 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 12 [next = 130, match = 129, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 129]
I161111 17:23:12.182844 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 11-132
I161111 17:23:12.211909 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:18 > next_replica_id:19
I161111 17:23:12.213940 1228 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:18}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.249174 452 storage/store.go:2988  [s3,r1/18:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:12.253284 340 storage/store.go:2041  [s3,r1/18:/M{in-ax}] DEL 1
I161111 17:23:12.253475 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 556d9c8b at index 135 in 64.882µs.
I161111 17:23:12.254357 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 186, log entries: 4
I161111 17:23:12.254602 1254 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.254658 1254 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:12.255138 1254 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.255155 1254 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:12.255179 1254 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:12.255192 1254 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:12.255552 1254 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 135, term: 6]
I161111 17:23:12.255572 1254 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 135, term: 6]
I161111 17:23:12.255591 1254 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 136, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.255608 1254 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 136, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.255620 1254 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 135] restored snapshot [index: 135, term: 6]
I161111 17:23:12.255640 1254 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 135 (id=556d9c8b, encoded size=16, 1 rocksdb batches, 4 log entries)
E161111 17:23:12.256486 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:12.265309 1254 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.010s
I161111 17:23:12.265354 1254 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:12.265603 241 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 12 became follower at term 6
I161111 17:23:12.265642 241 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 12 [peers: [1,2], term: 6, commit: 135, applied: 135, lastindex: 135, lastterm: 6]
I161111 17:23:12.266346 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:19
I161111 17:23:12.267825 1178 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:19}]
I161111 17:23:12.292059 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:12.309028 244 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 13 became follower at term 6
I161111 17:23:12.309075 244 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 13 [peers: [1,2], term: 6, commit: 135, applied: 135, lastindex: 135, lastterm: 6]
I161111 17:23:12.337850 245 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 13 [logterm: 0, index: 137] rejected msgApp [logterm: 6, index: 137] from 1
I161111 17:23:12.338056 71 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 135) from 13 for index 137
I161111 17:23:12.338081 71 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 13 to [next = 136, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.373877 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:19 > next_replica_id:20
I161111 17:23:12.376351 1160 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:19}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.379124 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 5adb7672 at index 140 in 78.838µs.
I161111 17:23:12.399408 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 193, log entries: 9
I161111 17:23:12.399737 340 storage/store.go:2041  [s3,r1/19:/M{in-ax}] DEL 1
I161111 17:23:12.401680 1272 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.401807 1272 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:12.402579 1272 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.402610 1272 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:12.402671 1272 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:12.402697 1272 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:12.404485 1272 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 140, term: 6]
I161111 17:23:12.404528 1272 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 140, term: 6]
I161111 17:23:12.404583 1272 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 141, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.404608 1272 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 141, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.404650 1272 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 140] restored snapshot [index: 140, term: 6]
I161111 17:23:12.404678 1272 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 140 (id=5adb7672, encoded size=16, 1 rocksdb batches, 9 log entries)
E161111 17:23:12.434914 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:12.434971 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:12.435029 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:12.435258 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:12.440172 1272 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.035s
I161111 17:23:12.440394 1272 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:12.441311 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:20
I161111 17:23:12.442864 1283 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:20}]
I161111 17:23:12.520009 256 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 14 became follower at term 6
I161111 17:23:12.520053 256 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 14 [peers: [1,2], term: 6, commit: 140, applied: 140, lastindex: 140, lastterm: 6]
I161111 17:23:12.520096 256 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 14 [logterm: 0, index: 144] rejected msgApp [logterm: 6, index: 144] from 1
I161111 17:23:12.520299 73 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 140) from 14 for index 144
I161111 17:23:12.520325 73 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 14 to [next = 141, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.542907 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:20 > next_replica_id:21
I161111 17:23:12.557766 1288 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:20}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.559366 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 91bbc2d8 at index 149 in 46.547µs.
I161111 17:23:12.560628 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 204, log entries: 18
I161111 17:23:12.560786 1336 raft/raft.go:568  [s3,r1/20:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:12.560827 1336 raft/raft.go:316  [s3,r1/20:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,14], term: 6, commit: 148, applied: 148, lastindex: 148, lastterm: 6]
I161111 17:23:12.560884 1336 raft/raft.go:1125  [s3,r1/20:/M{in-ax}] ffffffffffffffff [commit: 148, lastindex: 148, lastterm: 6] starts to restore snapshot [index: 149, term: 6]
I161111 17:23:12.560901 1336 raft/log.go:288  [s3,r1/20:/M{in-ax}] log [committed=148, applied=148, unstable.offset=149, len(unstable.Entries)=0] starts to restore snapshot [index: 149, term: 6]
I161111 17:23:12.560919 1336 raft/raft.go:1135  [s3,r1/20:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 150, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.560934 1336 raft/raft.go:1135  [s3,r1/20:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 150, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.560947 1336 raft/raft.go:1102  [s3,r1/20:/M{in-ax}] ffffffffffffffff [commit: 149] restored snapshot [index: 149, term: 6]
I161111 17:23:12.560964 1336 storage/replica_raftstorage.go:589  [s3,r1/20:/M{in-ax}] applying Raft snapshot at index 149 (id=91bbc2d8, encoded size=16, 1 rocksdb batches, 18 log entries)
I161111 17:23:12.564811 1336 storage/replica_raftstorage.go:592  [s3,r1/20:/M{in-ax}] applied Raft snapshot in 0.004s
I161111 17:23:12.565066 268 raft/raft.go:568  [s3,r1/20:/M{in-ax}] 14 became follower at term 6
I161111 17:23:12.565208 268 raft/raft.go:316  [s3,r1/20:/M{in-ax}] newRaft 14 [peers: [1,2], term: 6, commit: 149, applied: 149, lastindex: 149, lastterm: 6]
I161111 17:23:12.566024 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:21
I161111 17:23:12.566766 452 storage/store.go:2988  [s3,r1/20:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:12.569245 340 storage/store.go:2041  [s3,r1/20:/M{in-ax}] DEL 1
I161111 17:23:12.571856 1290 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:21}]
I161111 17:23:12.655702 270 raft/raft.go:568  [s3,r1/21:{-}] 15 became follower at term 0
I161111 17:23:12.656610 270 raft/raft.go:316  [s3,r1/21:{-}] newRaft 15 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.656630 270 raft/raft.go:568  [s3,r1/21:{-}] 15 became follower at term 1
I161111 17:23:12.656681 270 raft/raft.go:710  [s3,r1/21:{-}] 15 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:12.656695 270 raft/raft.go:568  [s3,r1/21:{-}] 15 became follower at term 6
I161111 17:23:12.691289 273 raft/raft.go:1088  [s3,r1/21:{-}] 15 [logterm: 0, index: 151] rejected msgApp [logterm: 6, index: 151] from 1
I161111 17:23:12.691425 110 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 15 for index 151
I161111 17:23:12.691451 110 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 15 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.691581 110 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 610d5ca9 at index 151 in 83.529µs.
I161111 17:23:12.691617 110 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 132, commit: 151] sent snapshot[index: 151, term: 6] to 15 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.691633 110 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 15 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 151]
I161111 17:23:12.692500 1261 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 207, log entries: 20
I161111 17:23:12.692613 1264 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.693456 1264 raft/raft.go:1125  [s3,r1/21:{-}] 15 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 151, term: 6]
I161111 17:23:12.693477 1264 raft/log.go:288  [s3,r1/21:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 151, term: 6]
I161111 17:23:12.693497 1264 raft/raft.go:1135  [s3,r1/21:{-}] 15 restored progress of 1 [next = 152, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.693536 1264 raft/raft.go:1135  [s3,r1/21:{-}] 15 restored progress of 2 [next = 152, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.693552 1264 raft/raft.go:1135  [s3,r1/21:{-}] 15 restored progress of 15 [next = 152, match = 151, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.693566 1264 raft/raft.go:1102  [s3,r1/21:{-}] 15 [commit: 151] restored snapshot [index: 151, term: 6]
I161111 17:23:12.693606 1264 storage/replica_raftstorage.go:589  [s3,r1/21:{-}] applying Raft snapshot at index 151 (id=610d5ca9, encoded size=16, 1 rocksdb batches, 20 log entries)
I161111 17:23:12.700549 1264 storage/replica_raftstorage.go:592  [s3,r1/21:/M{in-ax}] applied Raft snapshot in 0.007s
I161111 17:23:12.700962 16 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 15 [next = 152, match = 151, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 151]
I161111 17:23:12.790787 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:21 > next_replica_id:22
I161111 17:23:12.833757 1355 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:21}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.835814 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 49b0f53c at index 156 in 54.125µs.
I161111 17:23:12.840417 452 storage/store.go:2988  [s3,r1/21:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:12.841079 340 storage/store.go:2041  [s3,r1/21:/M{in-ax}] DEL 1
I161111 17:23:12.842230 1241 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.842291 1241 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:12.842828 1241 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.842844 1241 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:12.842867 1241 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:12.842882 1241 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:12.843314 1241 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 156, term: 6]
I161111 17:23:12.843338 1241 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 156, term: 6]
I161111 17:23:12.843362 1241 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.843384 1241 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.843400 1241 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 156] restored snapshot [index: 156, term: 6]
I161111 17:23:12.843419 1241 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 156 (id=49b0f53c, encoded size=16, 1 rocksdb batches, 25 log entries)
I161111 17:23:12.845706 1241 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.002s
I161111 17:23:12.845724 1241 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:12.845950 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 213, log entries: 25
I161111 17:23:12.846579 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:22
I161111 17:23:12.856431 1380 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:22}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:22}]
I161111 17:23:12.882328 249 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 16 became follower at term 6
I161111 17:23:12.882373 249 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 16 [peers: [1,2], term: 6, commit: 156, applied: 156, lastindex: 156, lastterm: 6]
I161111 17:23:12.882417 249 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 16 [logterm: 0, index: 158] rejected msgApp [logterm: 6, index: 158] from 1
I161111 17:23:12.882647 107 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 156) from 16 for index 158
I161111 17:23:12.882685 107 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 16 to [next = 157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.884702 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:22 > next_replica_id:23
I161111 17:23:12.886728 1371 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:22}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:12.888806 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ec1a1554 at index 161 in 51.922µs.
I161111 17:23:12.895534 340 storage/store.go:2041  [s3,r1/22:/M{in-ax}] DEL 1
I161111 17:23:12.902560 1394 storage/store.go:1912  [s3] ADD 1
I161111 17:23:12.902624 1394 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:12.903270 1394 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:12.903287 1394 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:12.903309 1394 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:12.903320 1394 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:12.903700 1394 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 161, term: 6]
I161111 17:23:12.903716 1394 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 161, term: 6]
I161111 17:23:12.903734 1394 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 162, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.903748 1394 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 162, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:12.903759 1394 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 161] restored snapshot [index: 161, term: 6]
I161111 17:23:12.903775 1394 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 161 (id=ec1a1554, encoded size=16, 1 rocksdb batches, 30 log entries)
I161111 17:23:12.907562 1394 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161111 17:23:12.907597 1394 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:12.907973 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 220, log entries: 30
I161111 17:23:12.908846 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:23
I161111 17:23:12.912609 1249 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:23}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:23}]
I161111 17:23:13.053549 261 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 17 became follower at term 6
I161111 17:23:13.053598 261 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 17 [peers: [1,2], term: 6, commit: 161, applied: 161, lastindex: 161, lastterm: 6]
I161111 17:23:13.058434 258 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 17 [logterm: 0, index: 165] rejected msgApp [logterm: 6, index: 165] from 1
I161111 17:23:13.058587 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 161) from 17 for index 165
I161111 17:23:13.058611 91 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 17 to [next = 162, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.094446 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:23 > next_replica_id:24
I161111 17:23:13.107228 1361 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:23}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:13.109430 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9b2a86bd at index 170 in 53.354µs.
I161111 17:23:13.111058 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 231, log entries: 39
I161111 17:23:13.111976 452 storage/store.go:2988  [s3,r1/23:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:13.114137 340 storage/store.go:2041  [s3,r1/23:/M{in-ax}] DEL 1
I161111 17:23:13.114483 1478 storage/store.go:1912  [s3] ADD 1
I161111 17:23:13.114632 1478 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:13.115396 1478 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:13.115445 1478 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:13.115482 1478 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:13.115525 1478 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:13.116077 1478 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 170, term: 6]
I161111 17:23:13.116107 1478 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 170, term: 6]
I161111 17:23:13.116158 1478 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 171, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.116186 1478 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 171, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.116227 1478 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 170] restored snapshot [index: 170, term: 6]
I161111 17:23:13.116259 1478 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 170 (id=9b2a86bd, encoded size=16, 1 rocksdb batches, 39 log entries)
I161111 17:23:13.119250 1478 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161111 17:23:13.120142 1478 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:13.124542 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:24
I161111 17:23:13.127606 1437 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:24}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:24}]
I161111 17:23:13.137294 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:13.241093 280 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 18 became follower at term 6
I161111 17:23:13.241135 280 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 18 [peers: [1,2], term: 6, commit: 170, applied: 170, lastindex: 170, lastterm: 6]
I161111 17:23:13.241181 280 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 18 [logterm: 0, index: 172] rejected msgApp [logterm: 6, index: 172] from 1
I161111 17:23:13.241363 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 170) from 18 for index 172
I161111 17:23:13.241387 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 18 to [next = 171, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.276961 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:24 > next_replica_id:25
I161111 17:23:13.281572 1522 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:24}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:13.284086 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7d4f71f2 at index 177 in 54.613µs.
I161111 17:23:13.284352 452 storage/store.go:2988  [s3,r1/24:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:13.287041 340 storage/store.go:2041  [s3,r1/24:/M{in-ax}] DEL 1
I161111 17:23:13.290922 1510 storage/store.go:1912  [s3] ADD 1
I161111 17:23:13.290983 1510 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:13.291682 1510 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:13.291701 1510 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:13.291724 1510 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:13.291737 1510 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:13.292191 1510 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 177, term: 6]
I161111 17:23:13.292216 1510 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 177, term: 6]
I161111 17:23:13.292237 1510 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 178, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.292252 1510 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 178, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.292266 1510 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 177] restored snapshot [index: 177, term: 6]
I161111 17:23:13.292285 1510 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 177 (id=7d4f71f2, encoded size=16, 1 rocksdb batches, 46 log entries)
W161111 17:23:13.292783 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 24}: raft group deleted
I161111 17:23:13.299054 1510 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.007s
I161111 17:23:13.299091 1510 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:13.314758 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 240, log entries: 46
I161111 17:23:13.315902 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:25
I161111 17:23:13.318217 1527 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:25}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:25}]
I161111 17:23:13.423360 252 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 19 became follower at term 6
I161111 17:23:13.423404 252 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 19 [peers: [1,2], term: 6, commit: 177, applied: 177, lastindex: 177, lastterm: 6]
I161111 17:23:13.423450 252 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 19 [logterm: 0, index: 179] rejected msgApp [logterm: 6, index: 179] from 1
I161111 17:23:13.423606 106 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 177) from 19 for index 179
I161111 17:23:13.423630 106 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 19 to [next = 178, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.442425 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:25 > next_replica_id:26
I161111 17:23:13.464400 1554 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:25}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:13.467973 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot b99a2c05 at index 184 in 83.496µs.
I161111 17:23:13.506638 452 storage/store.go:2988  [s3,r1/25:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:13.507829 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 249, log entries: 53
I161111 17:23:13.595925 340 storage/store.go:2041  [s3,r1/25:/M{in-ax}] DEL 1
I161111 17:23:13.596343 1550 storage/store.go:1912  [s3] ADD 1
I161111 17:23:13.596435 1550 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:13.597214 1550 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:13.617315 1550 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:13.617390 1550 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:13.617422 1550 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:13.618151 1550 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 184, term: 6]
I161111 17:23:13.618183 1550 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 184, term: 6]
I161111 17:23:13.618214 1550 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 185, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.618242 1550 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 185, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.618263 1550 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 184] restored snapshot [index: 184, term: 6]
I161111 17:23:13.618292 1550 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 184 (id=b99a2c05, encoded size=16, 1 rocksdb batches, 53 log entries)
I161111 17:23:13.621611 1550 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161111 17:23:13.621648 1550 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:13.625105 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:26
I161111 17:23:13.627189 1577 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:26}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:26}]
I161111 17:23:13.754699 268 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 1a became follower at term 6
I161111 17:23:13.754756 268 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 1a [peers: [1,2], term: 6, commit: 184, applied: 184, lastindex: 184, lastterm: 6]
I161111 17:23:13.778453 269 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 1a [logterm: 0, index: 189] rejected msgApp [logterm: 6, index: 189] from 1
I161111 17:23:13.778646 95 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 184) from 1a for index 189
I161111 17:23:13.778670 95 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1a to [next = 185, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.798229 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:26 > next_replica_id:27
I161111 17:23:13.847423 1605 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:26}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:13.849150 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 45157d14 at index 194 in 51.842µs.
I161111 17:23:13.853553 452 storage/store.go:2988  [s3,r1/26:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:13.855570 340 storage/store.go:2041  [s3,r1/26:/M{in-ax}] DEL 1
W161111 17:23:13.867684 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 26}: raft group deleted
I161111 17:23:13.901786 1622 storage/store.go:1912  [s3] ADD 1
I161111 17:23:13.901849 1622 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:13.902618 1622 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:13.902633 1622 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:13.902654 1622 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:13.902666 1622 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:13.903159 1622 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 194, term: 6]
I161111 17:23:13.903176 1622 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 194, term: 6]
I161111 17:23:13.903196 1622 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 195, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.903221 1622 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 195, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.903234 1622 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 194] restored snapshot [index: 194, term: 6]
I161111 17:23:13.903253 1622 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 194 (id=45157d14, encoded size=16, 1 rocksdb batches, 63 log entries)
I161111 17:23:13.906752 1622 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.003s
I161111 17:23:13.906773 1622 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:13.911686 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 261, log entries: 63
I161111 17:23:13.914518 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:27
I161111 17:23:13.917573 1625 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:27}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:27}]
I161111 17:23:13.958463 285 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 1b became follower at term 6
I161111 17:23:13.958505 285 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 1b [peers: [1,2], term: 6, commit: 194, applied: 194, lastindex: 194, lastterm: 6]
I161111 17:23:13.958553 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 1b [logterm: 0, index: 196] rejected msgApp [logterm: 6, index: 196] from 1
I161111 17:23:13.958712 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 194) from 1b for index 196
I161111 17:23:13.958736 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1b to [next = 195, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:13.962992 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:27 > next_replica_id:28
I161111 17:23:13.973220 1628 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:27}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:13.976032 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot aaecd406 at index 199 in 87.001µs.
I161111 17:23:14.057509 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 268, log entries: 68
I161111 17:23:14.058195 1469 raft/raft.go:568  [s3,r1/27:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:14.058234 1469 raft/raft.go:316  [s3,r1/27:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 199, applied: 199, lastindex: 199, lastterm: 6]
I161111 17:23:14.058280 1469 raft/raft.go:1106  [s3,r1/27:/M{in-ax}] ffffffffffffffff [commit: 199] ignored snapshot [index: 199, term: 6]
I161111 17:23:14.059093 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:28
I161111 17:23:14.059498 452 storage/store.go:2988  [s3,r1/27:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.061540 1613 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:28}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:28}]
I161111 17:23:14.068015 340 storage/store.go:2041  [s3,r1/27:/M{in-ax}] DEL 1
I161111 17:23:14.138716 249 raft/raft.go:568  [s3,r1/28:{-}] 1c became follower at term 0
I161111 17:23:14.139594 249 raft/raft.go:316  [s3,r1/28:{-}] newRaft 1c [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.139612 249 raft/raft.go:568  [s3,r1/28:{-}] 1c became follower at term 1
I161111 17:23:14.139635 249 raft/raft.go:710  [s3,r1/28:{-}] 1c [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:14.139646 249 raft/raft.go:568  [s3,r1/28:{-}] 1c became follower at term 6
I161111 17:23:14.139663 249 raft/raft.go:1088  [s3,r1/28:{-}] 1c [logterm: 0, index: 203] rejected msgApp [logterm: 6, index: 203] from 1
I161111 17:23:14.139889 86 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 1c for index 203
I161111 17:23:14.139914 86 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1c to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.140031 86 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8c89346a at index 203 in 71.683µs.
I161111 17:23:14.140094 86 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 132, commit: 203] sent snapshot[index: 203, term: 6] to 1c [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.140127 86 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 1c [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 203]
I161111 17:23:14.142919 1675 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 273, log entries: 72
I161111 17:23:14.143008 1681 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.143783 1681 raft/raft.go:1125  [s3,r1/28:{-}] 1c [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 203, term: 6]
I161111 17:23:14.143801 1681 raft/log.go:288  [s3,r1/28:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 203, term: 6]
I161111 17:23:14.143818 1681 raft/raft.go:1135  [s3,r1/28:{-}] 1c restored progress of 1 [next = 204, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.143833 1681 raft/raft.go:1135  [s3,r1/28:{-}] 1c restored progress of 2 [next = 204, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.143847 1681 raft/raft.go:1135  [s3,r1/28:{-}] 1c restored progress of 1c [next = 204, match = 203, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.143859 1681 raft/raft.go:1102  [s3,r1/28:{-}] 1c [commit: 203] restored snapshot [index: 203, term: 6]
I161111 17:23:14.143882 1681 storage/replica_raftstorage.go:589  [s3,r1/28:{-}] applying Raft snapshot at index 203 (id=8c89346a, encoded size=16, 1 rocksdb batches, 72 log entries)
I161111 17:23:14.149833 1681 storage/replica_raftstorage.go:592  [s3,r1/28:/M{in-ax}] applied Raft snapshot in 0.006s
I161111 17:23:14.150915 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:28 > next_replica_id:29
I161111 17:23:14.152459 1004 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:28}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:14.154802 452 storage/store.go:2988  [s3,r1/28:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.157786 340 storage/store.go:2041  [s3,r1/28:/M{in-ax}] DEL 1
I161111 17:23:14.166468 1675 raft/raft.go:854  [s1,r1/1:/M{in-ax}] 1 no progress available for 1c
I161111 17:23:14.166592 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 99d8f425 at index 208 in 57.128µs.
I161111 17:23:14.169236 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 279, log entries: 77
I161111 17:23:14.187912 1009 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.188042 1009 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
E161111 17:23:14.189253 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:14.189671 1009 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.189734 1009 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:14.189780 1009 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:14.189813 1009 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:14.190457 1009 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 208, term: 6]
I161111 17:23:14.190503 1009 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 208, term: 6]
I161111 17:23:14.190548 1009 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 209, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.190589 1009 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 209, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.190624 1009 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 208] restored snapshot [index: 208, term: 6]
I161111 17:23:14.190665 1009 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 208 (id=99d8f425, encoded size=16, 1 rocksdb batches, 77 log entries)
I161111 17:23:14.196321 1009 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.006s
I161111 17:23:14.196391 1009 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:14.197402 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:29
I161111 17:23:14.199716 1457 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:29}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:29}]
I161111 17:23:14.320429 252 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 1d became follower at term 6
I161111 17:23:14.320617 252 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 1d [peers: [1,2], term: 6, commit: 208, applied: 208, lastindex: 208, lastterm: 6]
I161111 17:23:14.320763 252 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 1d [logterm: 0, index: 210] rejected msgApp [logterm: 6, index: 210] from 1
I161111 17:23:14.321055 72 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 208) from 1d for index 210
I161111 17:23:14.321114 72 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1d to [next = 209, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.344419 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:29 > next_replica_id:30
I161111 17:23:14.355485 1643 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:29}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:14.359956 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 787d3c78 at index 215 in 54.384µs.
I161111 17:23:14.361003 452 storage/store.go:2988  [s3,r1/29:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.364023 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 288, log entries: 84
I161111 17:23:14.375013 340 storage/store.go:2041  [s3,r1/29:/M{in-ax}] DEL 1
W161111 17:23:14.375227 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 29}: raft group deleted
I161111 17:23:14.375640 1702 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.375722 1702 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:14.376525 1702 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.376561 1702 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:14.376595 1702 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:14.376619 1702 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:14.377250 1702 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 215, term: 6]
I161111 17:23:14.377287 1702 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 215, term: 6]
I161111 17:23:14.377320 1702 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 216, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.377349 1702 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 216, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.377437 1702 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 215] restored snapshot [index: 215, term: 6]
I161111 17:23:14.377522 1702 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 215 (id=787d3c78, encoded size=16, 1 rocksdb batches, 84 log entries)
I161111 17:23:14.394570 1702 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.017s
I161111 17:23:14.397323 1702 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:14.400355 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:30
I161111 17:23:14.402111 1710 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:30}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:30}]
I161111 17:23:14.408856 268 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 1e became follower at term 6
I161111 17:23:14.408900 268 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 1e [peers: [1,2], term: 6, commit: 215, applied: 215, lastindex: 215, lastterm: 6]
I161111 17:23:14.437460 269 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 1e [logterm: 0, index: 217] rejected msgApp [logterm: 6, index: 217] from 1
I161111 17:23:14.437618 88 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 215) from 1e for index 217
I161111 17:23:14.437646 88 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1e to [next = 216, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.447491 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:30 > next_replica_id:31
I161111 17:23:14.449380 1648 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:30}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:14.458145 452 storage/store.go:2988  [s3,r1/30:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.461575 340 storage/store.go:2041  [s3,r1/30:/M{in-ax}] DEL 1
I161111 17:23:14.462160 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 28a45337 at index 219 in 53.529µs.
I161111 17:23:14.463483 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 294, log entries: 88
I161111 17:23:14.463746 1781 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.463798 1781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:14.464688 1781 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.464704 1781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:14.464726 1781 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:14.464737 1781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:14.465354 1781 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 219, term: 6]
I161111 17:23:14.465371 1781 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 219, term: 6]
I161111 17:23:14.465390 1781 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 220, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.465406 1781 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 220, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.465418 1781 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 219] restored snapshot [index: 219, term: 6]
I161111 17:23:14.465435 1781 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 219 (id=28a45337, encoded size=16, 1 rocksdb batches, 88 log entries)
E161111 17:23:14.493210 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:14.493267 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:14.500793 1781 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.035s
I161111 17:23:14.500886 1781 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:14.503800 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:31
I161111 17:23:14.522250 1794 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:31}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:31}]
I161111 17:23:14.603808 284 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 1f became follower at term 6
I161111 17:23:14.603854 284 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 1f [peers: [1,2], term: 6, commit: 219, applied: 219, lastindex: 219, lastterm: 6]
I161111 17:23:14.639621 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 1f [logterm: 0, index: 224] rejected msgApp [logterm: 6, index: 224] from 1
I161111 17:23:14.639785 95 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 219) from 1f for index 224
I161111 17:23:14.639816 95 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 1f to [next = 220, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.704251 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:31 > next_replica_id:32
I161111 17:23:14.707583 1741 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:31}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:14.712442 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 360ae71e at index 229 in 52.984µs.
I161111 17:23:14.714883 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 306, log entries: 98
I161111 17:23:14.715664 1743 raft/raft.go:568  [s3,r1/31:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:14.715700 1743 raft/raft.go:316  [s3,r1/31:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 229, applied: 229, lastindex: 229, lastterm: 6]
I161111 17:23:14.715745 1743 raft/raft.go:1106  [s3,r1/31:/M{in-ax}] ffffffffffffffff [commit: 229] ignored snapshot [index: 229, term: 6]
I161111 17:23:14.717226 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:32
I161111 17:23:14.717761 452 storage/store.go:2988  [s3,r1/31:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.719798 340 storage/store.go:2041  [s3,r1/31:/M{in-ax}] DEL 1
I161111 17:23:14.733221 1569 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:32}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:32}]
W161111 17:23:14.763027 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 31}: raft group deleted
I161111 17:23:14.809677 262 raft/raft.go:568  [s3,r1/32:{-}] 20 became follower at term 0
I161111 17:23:14.810631 262 raft/raft.go:316  [s3,r1/32:{-}] newRaft 20 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.810648 262 raft/raft.go:568  [s3,r1/32:{-}] 20 became follower at term 1
I161111 17:23:14.810671 262 raft/raft.go:710  [s3,r1/32:{-}] 20 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:14.810684 262 raft/raft.go:568  [s3,r1/32:{-}] 20 became follower at term 6
I161111 17:23:14.837421 264 raft/raft.go:1088  [s3,r1/32:{-}] 20 [logterm: 0, index: 231] rejected msgApp [logterm: 6, index: 231] from 1
I161111 17:23:14.837554 104 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 20 for index 231
I161111 17:23:14.837583 104 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 20 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.837715 104 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7bff2e4e at index 231 in 84.964µs.
I161111 17:23:14.837751 104 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 132, commit: 231] sent snapshot[index: 231, term: 6] to 20 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.837767 104 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 20 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 231]
I161111 17:23:14.839271 764 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 309, log entries: 100
I161111 17:23:14.839383 767 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.840363 767 raft/raft.go:1125  [s3,r1/32:{-}] 20 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 231, term: 6]
I161111 17:23:14.840382 767 raft/log.go:288  [s3,r1/32:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 231, term: 6]
I161111 17:23:14.840398 767 raft/raft.go:1135  [s3,r1/32:{-}] 20 restored progress of 1 [next = 232, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.840412 767 raft/raft.go:1135  [s3,r1/32:{-}] 20 restored progress of 2 [next = 232, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.840425 767 raft/raft.go:1135  [s3,r1/32:{-}] 20 restored progress of 20 [next = 232, match = 231, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.840436 767 raft/raft.go:1102  [s3,r1/32:{-}] 20 [commit: 231] restored snapshot [index: 231, term: 6]
I161111 17:23:14.840459 767 storage/replica_raftstorage.go:589  [s3,r1/32:{-}] applying Raft snapshot at index 231 (id=7bff2e4e, encoded size=16, 1 rocksdb batches, 100 log entries)
I161111 17:23:14.852437 767 storage/replica_raftstorage.go:592  [s3,r1/32:/M{in-ax}] applied Raft snapshot in 0.012s
I161111 17:23:14.855199 764 raft/raft.go:933  [s1,r1/1:/M{in-ax}] 1 snapshot succeeded, resumed sending replication messages to 20 [next = 232, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.910649 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:32 > next_replica_id:33
I161111 17:23:14.912550 1875 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:32}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:14.912946 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 132-235
I161111 17:23:14.926301 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 79b546a8 at index 236 in 55.553µs.
I161111 17:23:14.927791 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 315, log entries: 105
I161111 17:23:14.947851 452 storage/store.go:2988  [s3,r1/32:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:14.957037 340 storage/store.go:2041  [s3,r1/32:/M{in-ax}] DEL 1
W161111 17:23:14.957478 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 32}: raft group deleted
I161111 17:23:14.957898 1893 storage/store.go:1912  [s3] ADD 1
I161111 17:23:14.959484 1893 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:14.960531 1893 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:14.964159 1893 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:14.964201 1893 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:14.964226 1893 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:14.969131 1893 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 236, term: 6]
I161111 17:23:14.969171 1893 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 236, term: 6]
I161111 17:23:14.969207 1893 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.969237 1893 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:14.969505 1893 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 236] restored snapshot [index: 236, term: 6]
I161111 17:23:14.969540 1893 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 236 (id=79b546a8, encoded size=16, 1 rocksdb batches, 105 log entries)
I161111 17:23:15.073360 1893 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.104s
I161111 17:23:15.073516 1893 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:15.074426 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:33
I161111 17:23:15.075862 1815 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:33}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:33}]
I161111 17:23:15.167357 240 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 21 became follower at term 6
I161111 17:23:15.167403 240 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 21 [peers: [1,2], term: 6, commit: 236, applied: 236, lastindex: 236, lastterm: 6]
I161111 17:23:15.195467 243 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 21 [logterm: 0, index: 242] rejected msgApp [logterm: 6, index: 242] from 1
I161111 17:23:15.195629 71 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 236) from 21 for index 242
I161111 17:23:15.195658 71 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 21 to [next = 237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.228486 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:33 > next_replica_id:34
I161111 17:23:15.253312 1896 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:33}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:15.265313 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot cb4b158c at index 247 in 52.463µs.
I161111 17:23:15.269772 340 storage/store.go:2041  [s3,r1/33:/M{in-ax}] DEL 1
I161111 17:23:15.273992 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 327, log entries: 13
I161111 17:23:15.274303 1924 storage/store.go:1912  [s3] ADD 1
I161111 17:23:15.274429 1924 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:15.275797 1924 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:15.275844 1924 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:15.275877 1924 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:15.275913 1924 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:15.276847 1924 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 247, term: 6]
I161111 17:23:15.276897 1924 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 247, term: 6]
I161111 17:23:15.276932 1924 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 248, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.276984 1924 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 248, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.277010 1924 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 247] restored snapshot [index: 247, term: 6]
I161111 17:23:15.277055 1924 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 247 (id=cb4b158c, encoded size=16, 1 rocksdb batches, 13 log entries)
I161111 17:23:15.291886 1924 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.015s
I161111 17:23:15.291929 1924 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:15.314957 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:34
I161111 17:23:15.316431 1820 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:34}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:34}]
I161111 17:23:15.398462 266 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 22 became follower at term 6
I161111 17:23:15.398509 266 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 22 [peers: [1,2], term: 6, commit: 247, applied: 247, lastindex: 247, lastterm: 6]
I161111 17:23:15.398556 266 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 22 [logterm: 0, index: 249] rejected msgApp [logterm: 6, index: 249] from 1
I161111 17:23:15.398757 16 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 247) from 22 for index 249
I161111 17:23:15.398784 16 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 22 to [next = 248, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.402891 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:34 > next_replica_id:35
I161111 17:23:15.404780 1905 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:34}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:15.406880 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot c2de5432 at index 252 in 46.784µs.
I161111 17:23:15.413438 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 334, log entries: 18
I161111 17:23:15.413551 452 storage/store.go:2988  [s3,r1/34:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:15.421844 340 storage/store.go:2041  [s3,r1/34:/M{in-ax}] DEL 1
I161111 17:23:15.426050 1987 storage/store.go:1912  [s3] ADD 1
I161111 17:23:15.426183 1987 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:15.433118 1987 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:15.433185 1987 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:15.433219 1987 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:15.433239 1987 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:15.434292 1987 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 252, term: 6]
I161111 17:23:15.434327 1987 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 252, term: 6]
I161111 17:23:15.434359 1987 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 253, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.434386 1987 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 253, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.434409 1987 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 252] restored snapshot [index: 252, term: 6]
I161111 17:23:15.434437 1987 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 252 (id=c2de5432, encoded size=16, 1 rocksdb batches, 18 log entries)
I161111 17:23:15.438027 1987 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161111 17:23:15.438066 1987 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:15.438839 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:35
I161111 17:23:15.440437 2005 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:35}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:35}]
I161111 17:23:15.453474 274 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 22 became follower at term 6
I161111 17:23:15.453525 274 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 22 [peers: [1,2], term: 6, commit: 252, applied: 252, lastindex: 252, lastterm: 6]
I161111 17:23:15.488497 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:15.553556 280 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 23 became follower at term 6
I161111 17:23:15.553604 280 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 23 [peers: [1,2], term: 6, commit: 252, applied: 252, lastindex: 252, lastterm: 6]
I161111 17:23:15.581420 277 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 23 [logterm: 0, index: 256] rejected msgApp [logterm: 6, index: 256] from 1
I161111 17:23:15.581585 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 252) from 23 for index 256
I161111 17:23:15.581610 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 23 to [next = 253, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.719830 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:35 > next_replica_id:36
I161111 17:23:15.721687 1995 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:35}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:15.724434 452 storage/store.go:2988  [s3,r1/35:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:15.729996 340 storage/store.go:2041  [s3,r1/35:/M{in-ax}] DEL 1
I161111 17:23:15.739889 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 04694d17 at index 261 in 122.049µs.
I161111 17:23:15.741513 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 345, log entries: 27
I161111 17:23:15.747246 2034 storage/store.go:1912  [s3] ADD 1
I161111 17:23:15.747410 2034 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:15.771737 2034 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:15.771778 2034 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:15.771834 2034 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:15.771860 2034 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:15.773797 2034 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 261, term: 6]
I161111 17:23:15.773945 2034 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 261, term: 6]
I161111 17:23:15.774090 2034 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 262, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.774123 2034 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 262, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.774149 2034 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 261] restored snapshot [index: 261, term: 6]
I161111 17:23:15.774638 2034 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 261 (id=04694d17, encoded size=16, 1 rocksdb batches, 27 log entries)
I161111 17:23:15.855636 2034 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.081s
I161111 17:23:15.855667 2034 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:15.855831 253 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 23 became follower at term 6
I161111 17:23:15.859919 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:36
I161111 17:23:15.861538 2058 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:36}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:36}]
I161111 17:23:15.864854 253 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 23 [peers: [1,2], term: 6, commit: 261, applied: 261, lastindex: 261, lastterm: 6]
I161111 17:23:15.870455 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:15.926154 257 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 24 became follower at term 6
I161111 17:23:15.926201 257 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 24 [peers: [1,2], term: 6, commit: 261, applied: 261, lastindex: 261, lastterm: 6]
I161111 17:23:15.937692 258 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 24 [logterm: 0, index: 266] rejected msgApp [logterm: 6, index: 266] from 1
I161111 17:23:15.938783 90 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 261) from 24 for index 266
I161111 17:23:15.938808 90 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 24 to [next = 262, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.942712 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:36 > next_replica_id:37
I161111 17:23:15.944533 1757 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:36}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:15.946883 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0aa978c4 at index 269 in 53.074µs.
I161111 17:23:15.947639 452 storage/store.go:2988  [s3,r1/36:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:15.951512 340 storage/store.go:2041  [s3,r1/36:/M{in-ax}] DEL 1
W161111 17:23:15.953310 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 36}: raft group deleted
I161111 17:23:15.954083 2104 storage/store.go:1912  [s3] ADD 1
I161111 17:23:15.954152 2104 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:15.955267 2104 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:15.955287 2104 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:15.955311 2104 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:15.955325 2104 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:15.956130 2104 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 269, term: 6]
I161111 17:23:15.956156 2104 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 269, term: 6]
I161111 17:23:15.956177 2104 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 270, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.956195 2104 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 270, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:15.956209 2104 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 269] restored snapshot [index: 269, term: 6]
I161111 17:23:15.956229 2104 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 269 (id=0aa978c4, encoded size=16, 1 rocksdb batches, 35 log entries)
I161111 17:23:15.962421 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 355, log entries: 35
I161111 17:23:15.966378 2104 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.010s
I161111 17:23:15.969160 2104 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:15.970024 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:37
I161111 17:23:15.972069 2086 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:37}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:37}]
I161111 17:23:16.003778 271 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 25 became follower at term 6
I161111 17:23:16.003823 271 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 25 [peers: [1,2], term: 6, commit: 269, applied: 269, lastindex: 269, lastterm: 6]
I161111 17:23:16.037423 273 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 25 [logterm: 0, index: 273] rejected msgApp [logterm: 6, index: 273] from 1
I161111 17:23:16.037575 81 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 269) from 25 for index 273
I161111 17:23:16.037603 81 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 25 to [next = 270, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.054851 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:37 > next_replica_id:38
I161111 17:23:16.061280 2092 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:37}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:16.063605 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 5e5c68ba at index 275 in 46.08µs.
I161111 17:23:16.064309 452 storage/store.go:2988  [s3,r1/37:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:16.069352 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 363, log entries: 41
I161111 17:23:16.104876 340 storage/store.go:2041  [s3,r1/37:/M{in-ax}] DEL 1
W161111 17:23:16.105246 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 37}: raft group deleted
I161111 17:23:16.108563 1981 storage/store.go:1912  [s3] ADD 1
I161111 17:23:16.108653 1981 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:16.109855 1981 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:16.109886 1981 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:16.109920 1981 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:16.109939 1981 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:16.110751 1981 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 275, term: 6]
I161111 17:23:16.110784 1981 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 275, term: 6]
I161111 17:23:16.110815 1981 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 276, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.110840 1981 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 276, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.110862 1981 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 275] restored snapshot [index: 275, term: 6]
I161111 17:23:16.110890 1981 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 275 (id=5e5c68ba, encoded size=16, 1 rocksdb batches, 41 log entries)
I161111 17:23:16.116053 1981 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161111 17:23:16.116095 1981 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:16.118785 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:38
I161111 17:23:16.219297 2039 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:38}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:38}]
I161111 17:23:16.308351 287 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 26 became follower at term 6
I161111 17:23:16.308395 287 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 26 [peers: [1,2], term: 6, commit: 275, applied: 275, lastindex: 275, lastterm: 6]
I161111 17:23:16.308442 287 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 26 [logterm: 0, index: 280] rejected msgApp [logterm: 6, index: 280] from 1
I161111 17:23:16.308621 109 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 275) from 26 for index 280
I161111 17:23:16.308646 109 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 26 to [next = 276, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.360637 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:38 > next_replica_id:39
I161111 17:23:16.362550 2047 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:38}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:16.364896 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot f84b2639 at index 285 in 52.11µs.
I161111 17:23:16.365642 452 storage/store.go:2988  [s3,r1/38:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:16.370613 340 storage/store.go:2041  [s3,r1/38:/M{in-ax}] DEL 1
I161111 17:23:16.375339 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 375, log entries: 51
I161111 17:23:16.378193 1984 storage/store.go:1912  [s3] ADD 1
I161111 17:23:16.378363 1984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:16.380088 1984 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:16.380147 1984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:16.380182 1984 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:16.380228 1984 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:16.385073 1984 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 285, term: 6]
I161111 17:23:16.385111 1984 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 285, term: 6]
I161111 17:23:16.385145 1984 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.385355 1984 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.385384 1984 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 285] restored snapshot [index: 285, term: 6]
I161111 17:23:16.385414 1984 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 285 (id=f84b2639, encoded size=16, 1 rocksdb batches, 51 log entries)
E161111 17:23:16.387302 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:16.441199 1984 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.056s
I161111 17:23:16.441430 1984 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:16.442494 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:39
I161111 17:23:16.444038 2159 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:39}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:39}]
I161111 17:23:16.478393 260 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 27 became follower at term 6
I161111 17:23:16.478431 260 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 27 [peers: [1,2], term: 6, commit: 285, applied: 285, lastindex: 285, lastterm: 6]
I161111 17:23:16.478479 260 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 27 [logterm: 0, index: 287] rejected msgApp [logterm: 6, index: 287] from 1
I161111 17:23:16.478665 68 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 285) from 27 for index 287
I161111 17:23:16.478687 68 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 27 to [next = 286, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.492259 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:39 > next_replica_id:40
I161111 17:23:16.494154 2129 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:39}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:16.496379 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0b172e69 at index 290 in 50.916µs.
I161111 17:23:16.503907 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 382, log entries: 56
I161111 17:23:16.525410 452 storage/store.go:2988  [s3,r1/39:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:16.579087 340 storage/store.go:2041  [s3,r1/39:/M{in-ax}] DEL 1
I161111 17:23:16.585383 2227 storage/store.go:1912  [s3] ADD 1
I161111 17:23:16.585493 2227 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:16.586754 2227 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:16.586790 2227 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:16.586825 2227 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:16.586852 2227 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:16.587734 2227 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 290, term: 6]
I161111 17:23:16.587771 2227 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 290, term: 6]
I161111 17:23:16.587810 2227 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.587845 2227 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.587868 2227 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 290] restored snapshot [index: 290, term: 6]
I161111 17:23:16.587896 2227 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 290 (id=0b172e69, encoded size=16, 1 rocksdb batches, 56 log entries)
I161111 17:23:16.616879 2227 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.029s
I161111 17:23:16.616921 2227 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:16.618001 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:40
I161111 17:23:16.619656 2202 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:40}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:40}]
I161111 17:23:16.677176 275 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 28 became follower at term 6
I161111 17:23:16.677221 275 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 28 [peers: [1,2], term: 6, commit: 290, applied: 290, lastindex: 290, lastterm: 6]
I161111 17:23:16.677269 275 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 28 [logterm: 0, index: 294] rejected msgApp [logterm: 6, index: 294] from 1
I161111 17:23:16.678386 69 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 290) from 28 for index 294
I161111 17:23:16.678411 69 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 28 to [next = 291, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.693377 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:40 > next_replica_id:41
I161111 17:23:16.695228 2278 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:40}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:16.697368 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 311f6462 at index 298 in 52.801µs.
I161111 17:23:16.700233 452 storage/store.go:2988  [s3,r1/40:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:16.708491 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 392, log entries: 64
I161111 17:23:16.713379 2293 raft/raft.go:568  [s3,r1/40:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:16.713450 2293 raft/raft.go:316  [s3,r1/40:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 298, applied: 298, lastindex: 298, lastterm: 6]
I161111 17:23:16.713529 2293 raft/raft.go:1106  [s3,r1/40:/M{in-ax}] ffffffffffffffff [commit: 298] ignored snapshot [index: 298, term: 6]
I161111 17:23:16.721510 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:41
I161111 17:23:16.726451 340 storage/store.go:2041  [s3,r1/40:/M{in-ax}] DEL 1
I161111 17:23:16.727361 2245 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:41}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:41}]
I161111 17:23:16.808024 243 raft/raft.go:568  [s3,r1/41:{-}] 29 became follower at term 0
I161111 17:23:16.809246 243 raft/raft.go:316  [s3,r1/41:{-}] newRaft 29 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:16.809263 243 raft/raft.go:568  [s3,r1/41:{-}] 29 became follower at term 1
I161111 17:23:16.809284 243 raft/raft.go:710  [s3,r1/41:{-}] 29 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:16.809294 243 raft/raft.go:568  [s3,r1/41:{-}] 29 became follower at term 6
I161111 17:23:16.842154 245 raft/raft.go:1088  [s3,r1/41:{-}] 29 [logterm: 0, index: 301] rejected msgApp [logterm: 6, index: 301] from 1
I161111 17:23:16.843079 71 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 29 for index 301
I161111 17:23:16.843107 71 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 29 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.843222 71 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7ba249b9 at index 302 in 63.091µs.
I161111 17:23:16.843258 71 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 235, commit: 302] sent snapshot[index: 302, term: 6] to 29 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.843275 71 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 29 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 302]
I161111 17:23:16.846887 2230 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 397, log entries: 68
I161111 17:23:16.847260 2323 storage/store.go:1912  [s3] ADD 1
I161111 17:23:16.854119 2323 raft/raft.go:1125  [s3,r1/41:{-}] 29 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 302, term: 6]
I161111 17:23:16.854254 2323 raft/log.go:288  [s3,r1/41:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 302, term: 6]
I161111 17:23:16.854336 2323 raft/raft.go:1135  [s3,r1/41:{-}] 29 restored progress of 1 [next = 303, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.854430 2323 raft/raft.go:1135  [s3,r1/41:{-}] 29 restored progress of 2 [next = 303, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.854456 2323 raft/raft.go:1135  [s3,r1/41:{-}] 29 restored progress of 29 [next = 303, match = 302, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:16.854572 2323 raft/raft.go:1102  [s3,r1/41:{-}] 29 [commit: 302] restored snapshot [index: 302, term: 6]
I161111 17:23:16.854627 2323 storage/replica_raftstorage.go:589  [s3,r1/41:{-}] applying Raft snapshot at index 302 (id=7ba249b9, encoded size=16, 1 rocksdb batches, 68 log entries)
I161111 17:23:16.927856 2323 storage/replica_raftstorage.go:592  [s3,r1/41:/M{in-ax}] applied Raft snapshot in 0.073s
I161111 17:23:16.929243 89 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 29 [next = 303, match = 302, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 302]
I161111 17:23:17.045570 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:41 > next_replica_id:42
I161111 17:23:17.052136 2342 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:41}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.053756 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot d591f5bb at index 309 in 50.375µs.
I161111 17:23:17.057739 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 405, log entries: 75
I161111 17:23:17.057986 452 storage/store.go:2988  [s3,r1/41:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.067725 340 storage/store.go:2041  [s3,r1/41:/M{in-ax}] DEL 1
I161111 17:23:17.068202 2297 storage/store.go:1912  [s3] ADD 1
I161111 17:23:17.068269 2297 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:17.069608 2297 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:17.069627 2297 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:17.069649 2297 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:17.069661 2297 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:17.070530 2297 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 309, term: 6]
I161111 17:23:17.070554 2297 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 309, term: 6]
I161111 17:23:17.070577 2297 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 310, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.070592 2297 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 310, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.070605 2297 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 309] restored snapshot [index: 309, term: 6]
I161111 17:23:17.070624 2297 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 309 (id=d591f5bb, encoded size=16, 1 rocksdb batches, 75 log entries)
W161111 17:23:17.073897 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 41}: raft group deleted
I161111 17:23:17.077183 2297 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.007s
I161111 17:23:17.077241 2297 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:17.077979 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:42
I161111 17:23:17.079394 2256 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:42}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:42}]
I161111 17:23:17.157887 271 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2a became follower at term 6
I161111 17:23:17.157937 271 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2a [peers: [1,2], term: 6, commit: 309, applied: 309, lastindex: 309, lastterm: 6]
I161111 17:23:17.187482 273 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 2a [logterm: 0, index: 311] rejected msgApp [logterm: 6, index: 311] from 1
I161111 17:23:17.187732 96 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 309) from 2a for index 311
I161111 17:23:17.187763 96 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2a to [next = 310, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.234877 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:42 > next_replica_id:43
I161111 17:23:17.236751 2304 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:42}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.238837 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0d9c94d4 at index 316 in 52.168µs.
I161111 17:23:17.253811 452 storage/store.go:2988  [s3,r1/42:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.254971 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 414, log entries: 82
I161111 17:23:17.291559 340 storage/store.go:2041  [s3,r1/42:/M{in-ax}] DEL 1
I161111 17:23:17.291966 2234 storage/store.go:1912  [s3] ADD 1
I161111 17:23:17.292049 2234 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:17.331834 2234 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:17.331887 2234 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:17.331927 2234 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:17.331953 2234 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:17.334584 2234 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 316, term: 6]
I161111 17:23:17.334640 2234 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 316, term: 6]
I161111 17:23:17.335098 2234 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 317, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.335145 2234 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 317, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.335184 2234 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 316] restored snapshot [index: 316, term: 6]
I161111 17:23:17.335232 2234 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 316 (id=0d9c94d4, encoded size=16, 1 rocksdb batches, 82 log entries)
I161111 17:23:17.342824 2234 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.008s
I161111 17:23:17.342867 2234 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:17.344464 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:43
I161111 17:23:17.346017 2434 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:43}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:43}]
I161111 17:23:17.381363 250 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2b became follower at term 6
I161111 17:23:17.381411 250 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2b [peers: [1,2], term: 6, commit: 316, applied: 316, lastindex: 316, lastterm: 6]
I161111 17:23:17.381462 250 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 2b [logterm: 0, index: 318] rejected msgApp [logterm: 6, index: 318] from 1
I161111 17:23:17.381629 95 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 316) from 2b for index 318
I161111 17:23:17.381657 95 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2b to [next = 317, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.401989 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:43 > next_replica_id:44
I161111 17:23:17.404059 2443 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:43}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.407627 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8568d7d5 at index 322 in 48.256µs.
I161111 17:23:17.408949 452 storage/store.go:2988  [s3,r1/43:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.413768 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 422, log entries: 88
I161111 17:23:17.416812 340 storage/store.go:2041  [s3,r1/43:/M{in-ax}] DEL 1
W161111 17:23:17.417001 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 43}: raft group deleted
I161111 17:23:17.417252 1770 storage/store.go:1912  [s3] ADD 1
I161111 17:23:17.417351 1770 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:17.418693 1770 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:17.418726 1770 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:17.418762 1770 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:17.418788 1770 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:17.419783 1770 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 322, term: 6]
I161111 17:23:17.419853 1770 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 322, term: 6]
I161111 17:23:17.419887 1770 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.419918 1770 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.419943 1770 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 322] restored snapshot [index: 322, term: 6]
I161111 17:23:17.419971 1770 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 322 (id=8568d7d5, encoded size=16, 1 rocksdb batches, 88 log entries)
I161111 17:23:17.433455 1770 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.013s
I161111 17:23:17.502663 1770 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:17.503207 267 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2b became follower at term 6
I161111 17:23:17.503257 267 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2b [peers: [1,2], term: 6, commit: 322, applied: 322, lastindex: 322, lastterm: 6]
I161111 17:23:17.503693 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:44
I161111 17:23:17.505436 2477 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:44}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:44}]
I161111 17:23:17.538192 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.561081 268 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2c became follower at term 6
I161111 17:23:17.561126 268 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2c [peers: [1,2], term: 6, commit: 322, applied: 322, lastindex: 322, lastterm: 6]
I161111 17:23:17.561171 268 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 2c [logterm: 0, index: 325] rejected msgApp [logterm: 6, index: 325] from 1
I161111 17:23:17.561333 102 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 322) from 2c for index 325
I161111 17:23:17.561383 102 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2c to [next = 323, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.588672 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:44 > next_replica_id:45
I161111 17:23:17.590967 2456 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:44}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.598251 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8ccba653 at index 330 in 51.856µs.
I161111 17:23:17.625260 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 432, log entries: 96
I161111 17:23:17.694999 340 storage/store.go:2041  [s3,r1/44:/M{in-ax}] DEL 1
I161111 17:23:17.695593 2415 storage/store.go:1912  [s3] ADD 1
I161111 17:23:17.695738 2415 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:17.697891 2415 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:17.697947 2415 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:17.697998 2415 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:17.698025 2415 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:17.699371 2415 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 330, term: 6]
I161111 17:23:17.699407 2415 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 330, term: 6]
I161111 17:23:17.699453 2415 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 331, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.699487 2415 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 331, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.699508 2415 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 330] restored snapshot [index: 330, term: 6]
I161111 17:23:17.699550 2415 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 330 (id=8ccba653, encoded size=16, 1 rocksdb batches, 96 log entries)
E161111 17:23:17.710146 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:17.733700 2415 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.034s
I161111 17:23:17.746355 2415 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:17.749589 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:45
I161111 17:23:17.751696 2423 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:45}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:45}]
I161111 17:23:17.756420 285 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2d became follower at term 6
I161111 17:23:17.756462 285 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2d [peers: [1,2], term: 6, commit: 330, applied: 330, lastindex: 330, lastterm: 6]
I161111 17:23:17.756517 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 2d [logterm: 0, index: 333] rejected msgApp [logterm: 6, index: 333] from 1
I161111 17:23:17.756687 99 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 330) from 2d for index 333
I161111 17:23:17.756710 99 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2d to [next = 331, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.761864 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:45 > next_replica_id:46
I161111 17:23:17.764587 2531 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:45}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.773609 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 19edbc80 at index 337 in 57.978µs.
I161111 17:23:17.777875 452 storage/store.go:2988  [s3,r1/45:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.827304 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 441, log entries: 103
I161111 17:23:17.828281 340 storage/store.go:2041  [s3,r1/45:/M{in-ax}] DEL 1
I161111 17:23:17.828565 2464 storage/store.go:1912  [s3] ADD 1
I161111 17:23:17.828645 2464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:17.830022 2464 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:17.830054 2464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:17.830087 2464 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:17.830109 2464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:17.831068 2464 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 337, term: 6]
I161111 17:23:17.831103 2464 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 337, term: 6]
I161111 17:23:17.831135 2464 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.831162 2464 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.831184 2464 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 337] restored snapshot [index: 337, term: 6]
I161111 17:23:17.831211 2464 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 337 (id=19edbc80, encoded size=16, 1 rocksdb batches, 103 log entries)
I161111 17:23:17.870492 2464 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.039s
I161111 17:23:17.872261 2464 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:17.872991 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:46
I161111 17:23:17.874549 2551 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:46}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:46}]
I161111 17:23:17.876452 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 235-339
I161111 17:23:17.878111 254 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 2e became follower at term 6
I161111 17:23:17.878156 254 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 2e [peers: [1,2], term: 6, commit: 337, applied: 337, lastindex: 337, lastterm: 6]
I161111 17:23:17.878198 254 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 2e [logterm: 0, index: 339] rejected msgApp [logterm: 6, index: 339] from 1
I161111 17:23:17.880640 89 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 337) from 2e for index 339
I161111 17:23:17.880670 89 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2e to [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.880770 89 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1a222cd1 at index 340 in 55.58µs.
I161111 17:23:17.894765 89 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 339, commit: 340] sent snapshot[index: 340, term: 6] to 2e [next = 338, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.894848 89 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 2e [next = 338, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 340]
I161111 17:23:17.899697 1776 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 444, log entries: 2
I161111 17:23:17.899907 2565 raft/raft.go:1125  [s3,r1/?:/M{in-ax}] 2e [commit: 337, lastindex: 337, lastterm: 6] starts to restore snapshot [index: 340, term: 6]
I161111 17:23:17.899941 2565 raft/log.go:288  [s3,r1/?:/M{in-ax}] log [committed=337, applied=337, unstable.offset=338, len(unstable.Entries)=0] starts to restore snapshot [index: 340, term: 6]
I161111 17:23:17.899977 2565 raft/raft.go:1135  [s3,r1/?:/M{in-ax}] 2e restored progress of 1 [next = 341, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.900005 2565 raft/raft.go:1135  [s3,r1/?:/M{in-ax}] 2e restored progress of 2 [next = 341, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.900030 2565 raft/raft.go:1135  [s3,r1/?:/M{in-ax}] 2e restored progress of 2e [next = 341, match = 340, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.900050 2565 raft/raft.go:1102  [s3,r1/?:/M{in-ax}] 2e [commit: 340] restored snapshot [index: 340, term: 6]
I161111 17:23:17.900088 2565 storage/replica_raftstorage.go:589  [s3,r1/?:/M{in-ax}] applying Raft snapshot at index 340 (id=1a222cd1, encoded size=16, 1 rocksdb batches, 2 log entries)
I161111 17:23:17.943323 2565 storage/replica_raftstorage.go:592  [s3,r1/46:/M{in-ax}] applied Raft snapshot in 0.043s
I161111 17:23:17.943805 101 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 2e [next = 341, match = 340, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 340]
I161111 17:23:17.946125 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:46 > next_replica_id:47
I161111 17:23:17.951733 2573 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:46}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:17.954739 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot fadea738 at index 345 in 51.321µs.
I161111 17:23:17.957356 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 450, log entries: 7
I161111 17:23:17.957855 2593 raft/raft.go:568  [s3,r1/46:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:17.957932 2593 raft/raft.go:316  [s3,r1/46:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,2e], term: 6, commit: 340, applied: 340, lastindex: 341, lastterm: 6]
I161111 17:23:17.958033 2593 raft/raft.go:1125  [s3,r1/46:/M{in-ax}] ffffffffffffffff [commit: 340, lastindex: 341, lastterm: 6] starts to restore snapshot [index: 345, term: 6]
I161111 17:23:17.958089 2593 raft/log.go:288  [s3,r1/46:/M{in-ax}] log [committed=340, applied=340, unstable.offset=342, len(unstable.Entries)=0] starts to restore snapshot [index: 345, term: 6]
I161111 17:23:17.958120 2593 raft/raft.go:1135  [s3,r1/46:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 346, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.958170 2593 raft/raft.go:1135  [s3,r1/46:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 346, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:17.958198 452 storage/store.go:2988  [s3,r1/46:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:17.958296 2593 raft/raft.go:1102  [s3,r1/46:/M{in-ax}] ffffffffffffffff [commit: 345] restored snapshot [index: 345, term: 6]
I161111 17:23:17.958328 2593 storage/replica_raftstorage.go:589  [s3,r1/46:/M{in-ax}] applying Raft snapshot at index 345 (id=fadea738, encoded size=16, 1 rocksdb batches, 7 log entries)
I161111 17:23:17.975882 2593 storage/replica_raftstorage.go:592  [s3,r1/46:/M{in-ax}] applied Raft snapshot in 0.018s
I161111 17:23:17.978280 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:47
I161111 17:23:17.980004 2543 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:47}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:47}]
I161111 17:23:17.990745 452 storage/store.go:2988  [s3,r1/46:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:18.035212 340 storage/store.go:2041  [s3,r1/46:/M{in-ax}] DEL 1
I161111 17:23:18.099231 264 raft/raft.go:568  [s3,r1/47:{-}] 2f became follower at term 0
I161111 17:23:18.100697 264 raft/raft.go:316  [s3,r1/47:{-}] newRaft 2f [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:18.107647 264 raft/raft.go:568  [s3,r1/47:{-}] 2f became follower at term 1
I161111 17:23:18.107692 264 raft/raft.go:710  [s3,r1/47:{-}] 2f [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:18.107705 264 raft/raft.go:568  [s3,r1/47:{-}] 2f became follower at term 6
I161111 17:23:18.107723 264 raft/raft.go:1088  [s3,r1/47:{-}] 2f [logterm: 0, index: 347] rejected msgApp [logterm: 6, index: 347] from 1
I161111 17:23:18.111509 76 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 2f for index 347
I161111 17:23:18.111550 76 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 2f to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.111681 76 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a79d4d28 at index 348 in 77.054µs.
I161111 17:23:18.111720 76 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 339, commit: 348] sent snapshot[index: 348, term: 6] to 2f [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.111739 76 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 2f [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 348]
I161111 17:23:18.114536 2616 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 454, log entries: 10
I161111 17:23:18.114672 2600 storage/store.go:1912  [s3] ADD 1
I161111 17:23:18.116203 2600 raft/raft.go:1125  [s3,r1/47:{-}] 2f [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 348, term: 6]
I161111 17:23:18.116225 2600 raft/log.go:288  [s3,r1/47:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 348, term: 6]
I161111 17:23:18.116245 2600 raft/raft.go:1135  [s3,r1/47:{-}] 2f restored progress of 1 [next = 349, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.116260 2600 raft/raft.go:1135  [s3,r1/47:{-}] 2f restored progress of 2 [next = 349, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.116274 2600 raft/raft.go:1135  [s3,r1/47:{-}] 2f restored progress of 2f [next = 349, match = 348, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.116285 2600 raft/raft.go:1102  [s3,r1/47:{-}] 2f [commit: 348] restored snapshot [index: 348, term: 6]
I161111 17:23:18.116314 2600 storage/replica_raftstorage.go:589  [s3,r1/47:{-}] applying Raft snapshot at index 348 (id=a79d4d28, encoded size=16, 1 rocksdb batches, 10 log entries)
I161111 17:23:18.179559 2600 storage/replica_raftstorage.go:592  [s3,r1/47:/M{in-ax}] applied Raft snapshot in 0.063s
I161111 17:23:18.179969 87 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 2f [next = 349, match = 348, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 348]
I161111 17:23:18.264816 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:47 > next_replica_id:48
I161111 17:23:18.266742 2608 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:47}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:18.269205 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3ee80f68 at index 352 in 53.567µs.
I161111 17:23:18.270207 452 storage/store.go:2988  [s3,r1/47:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:18.275136 340 storage/store.go:2041  [s3,r1/47:/M{in-ax}] DEL 1
W161111 17:23:18.276193 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 47}: raft group deleted
I161111 17:23:18.342614 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 459, log entries: 14
I161111 17:23:18.343051 2560 storage/store.go:1912  [s3] ADD 1
I161111 17:23:18.343183 2560 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:18.345169 2560 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:18.345228 2560 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:18.345264 2560 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:18.345308 2560 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:18.346691 2560 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 352, term: 6]
I161111 17:23:18.346745 2560 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 352, term: 6]
I161111 17:23:18.346778 2560 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 353, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.346824 2560 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 353, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.346847 2560 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 352] restored snapshot [index: 352, term: 6]
I161111 17:23:18.346894 2560 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 352 (id=3ee80f68, encoded size=16, 1 rocksdb batches, 14 log entries)
I161111 17:23:18.369253 2560 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.022s
I161111 17:23:18.369295 2560 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:18.370001 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:48
I161111 17:23:18.371517 2661 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:48}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:48}]
I161111 17:23:18.453715 277 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 30 became follower at term 6
I161111 17:23:18.453764 277 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 30 [peers: [1,2], term: 6, commit: 352, applied: 352, lastindex: 352, lastterm: 6]
I161111 17:23:18.505100 282 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 30 [logterm: 0, index: 357] rejected msgApp [logterm: 6, index: 357] from 1
I161111 17:23:18.505315 83 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 352) from 30 for index 357
I161111 17:23:18.505357 83 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 30 to [next = 353, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.691134 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:48 > next_replica_id:49
I161111 17:23:18.693076 2748 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:48}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:18.697453 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 699d2bba at index 365 in 1.930715ms.
I161111 17:23:18.698304 452 storage/store.go:2988  [s3,r1/48:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:18.699453 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 474, log entries: 27
I161111 17:23:18.699644 2696 raft/raft.go:568  [s3,r1/48:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:18.699678 2696 raft/raft.go:316  [s3,r1/48:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 365, applied: 365, lastindex: 365, lastterm: 6]
I161111 17:23:18.699721 2696 raft/raft.go:1106  [s3,r1/48:/M{in-ax}] ffffffffffffffff [commit: 365] ignored snapshot [index: 365, term: 6]
I161111 17:23:18.700312 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:49
I161111 17:23:18.702328 2677 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:49}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:49}]
I161111 17:23:18.706050 270 raft/raft.go:568  [s3,r1/48:/M{in-ax}] 31 became follower at term 6
I161111 17:23:18.706136 270 raft/raft.go:316  [s3,r1/48:/M{in-ax}] newRaft 31 [peers: [1,2], term: 6, commit: 365, applied: 365, lastindex: 365, lastterm: 6]
I161111 17:23:18.737491 269 raft/raft.go:1088  [s3,r1/48:/M{in-ax}] 31 [logterm: 0, index: 367] rejected msgApp [logterm: 6, index: 367] from 1
I161111 17:23:18.737636 69 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 365) from 31 for index 367
I161111 17:23:18.737665 69 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 31 to [next = 366, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.748721 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:49 > next_replica_id:50
I161111 17:23:18.753920 2788 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:49}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:18.759156 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot e22f5bf4 at index 369 in 60.066µs.
I161111 17:23:18.764049 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 480, log entries: 31
I161111 17:23:18.768246 340 storage/store.go:2041  [s3,r1/49:/M{in-ax}] DEL 1
I161111 17:23:18.768723 2679 storage/store.go:1912  [s3] ADD 1
I161111 17:23:18.768836 2679 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:18.793879 2679 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:18.793941 2679 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:18.793980 2679 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:18.794025 2679 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:18.797192 2679 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 369, term: 6]
I161111 17:23:18.797329 2679 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 369, term: 6]
I161111 17:23:18.797366 2679 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 370, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.797498 2679 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 370, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.797524 2679 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 369] restored snapshot [index: 369, term: 6]
I161111 17:23:18.797886 2679 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 369 (id=e22f5bf4, encoded size=16, 1 rocksdb batches, 31 log entries)
I161111 17:23:18.830660 2679 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.033s
I161111 17:23:18.830845 2679 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:18.831654 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:50
I161111 17:23:18.833076 2683 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:50}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:50}]
I161111 17:23:18.855746 283 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 32 became follower at term 6
I161111 17:23:18.855812 283 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 32 [peers: [1,2], term: 6, commit: 369, applied: 369, lastindex: 369, lastterm: 6]
I161111 17:23:18.855885 283 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 32 [logterm: 0, index: 372] rejected msgApp [logterm: 6, index: 372] from 1
I161111 17:23:18.856179 88 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 369) from 32 for index 372
I161111 17:23:18.856218 88 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 32 to [next = 370, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.874081 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:50 > next_replica_id:51
I161111 17:23:18.876889 2811 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:50}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:18.878858 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 569bb984 at index 376 in 55.139µs.
I161111 17:23:18.887490 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 489, log entries: 38
I161111 17:23:18.896447 452 storage/store.go:2988  [s3,r1/50:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:18.897037 340 storage/store.go:2041  [s3,r1/50:/M{in-ax}] DEL 1
I161111 17:23:18.897418 2714 storage/store.go:1912  [s3] ADD 1
I161111 17:23:18.897497 2714 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:18.899055 2714 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:18.899075 2714 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:18.899098 2714 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:18.899111 2714 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:18.900166 2714 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 376, term: 6]
I161111 17:23:18.900192 2714 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 376, term: 6]
I161111 17:23:18.900215 2714 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 377, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.900232 2714 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 377, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:18.900246 2714 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 376] restored snapshot [index: 376, term: 6]
I161111 17:23:18.900265 2714 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 376 (id=569bb984, encoded size=16, 1 rocksdb batches, 38 log entries)
I161111 17:23:18.926985 2714 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.027s
I161111 17:23:18.927024 2714 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:18.928341 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:51
I161111 17:23:18.930391 2820 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:51}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:51}]
I161111 17:23:19.002496 246 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 33 became follower at term 6
I161111 17:23:19.002541 246 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 33 [peers: [1,2], term: 6, commit: 376, applied: 376, lastindex: 376, lastterm: 6]
I161111 17:23:19.002587 246 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 33 [logterm: 0, index: 378] rejected msgApp [logterm: 6, index: 378] from 1
I161111 17:23:19.002773 84 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 376) from 33 for index 378
I161111 17:23:19.002797 84 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 33 to [next = 377, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.010128 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:51 > next_replica_id:52
I161111 17:23:19.013187 2718 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:51}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:19.015430 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3fe0d0a1 at index 381 in 49.146µs.
I161111 17:23:19.025268 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 496, log entries: 43
I161111 17:23:19.025700 452 storage/store.go:2988  [s3,r1/51:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:19.034609 340 storage/store.go:2041  [s3,r1/51:/M{in-ax}] DEL 1
I161111 17:23:19.108159 2720 storage/store.go:1912  [s3] ADD 1
I161111 17:23:19.108540 2720 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:19.113699 2720 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:19.120223 2720 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
E161111 17:23:19.120365 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:19.120564 2720 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:19.120648 2720 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:19.124632 2720 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 381, term: 6]
I161111 17:23:19.124669 2720 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 381, term: 6]
I161111 17:23:19.124706 2720 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 382, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.124734 2720 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 382, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.124846 2720 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 381] restored snapshot [index: 381, term: 6]
I161111 17:23:19.127254 2720 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 381 (id=3fe0d0a1, encoded size=16, 1 rocksdb batches, 43 log entries)
I161111 17:23:19.229657 2720 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.102s
I161111 17:23:19.271228 2720 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:19.272547 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:52
I161111 17:23:19.274113 2890 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:52}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:52}]
I161111 17:23:19.353746 259 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 34 became follower at term 6
I161111 17:23:19.353798 259 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 34 [peers: [1,2], term: 6, commit: 381, applied: 381, lastindex: 381, lastterm: 6]
I161111 17:23:19.360784 263 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 34 [logterm: 0, index: 388] rejected msgApp [logterm: 6, index: 388] from 1
I161111 17:23:19.361888 98 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 381) from 34 for index 388
I161111 17:23:19.361916 98 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 34 to [next = 382, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.382748 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:52 > next_replica_id:53
I161111 17:23:19.385130 2921 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:52}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:19.387915 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8eb5d54f at index 393 in 53.351µs.
I161111 17:23:19.393693 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 510, log entries: 55
I161111 17:23:19.393894 452 storage/store.go:2988  [s3,r1/52:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:19.415485 340 storage/store.go:2041  [s3,r1/52:/M{in-ax}] DEL 1
I161111 17:23:19.416967 2657 storage/store.go:1912  [s3] ADD 1
I161111 17:23:19.417053 2657 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:19.418651 2657 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:19.418692 2657 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:19.418736 2657 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:19.418794 2657 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:19.419891 2657 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 393, term: 6]
I161111 17:23:19.419926 2657 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 393, term: 6]
I161111 17:23:19.419959 2657 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 394, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.419986 2657 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 394, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.420010 2657 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 393] restored snapshot [index: 393, term: 6]
I161111 17:23:19.420037 2657 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 393 (id=8eb5d54f, encoded size=16, 1 rocksdb batches, 55 log entries)
I161111 17:23:19.453079 2657 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.033s
I161111 17:23:19.453313 2657 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:19.454384 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:53
I161111 17:23:19.455913 2931 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:53}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:53}]
I161111 17:23:19.543035 279 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 35 became follower at term 6
I161111 17:23:19.543085 279 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 35 [peers: [1,2], term: 6, commit: 393, applied: 393, lastindex: 393, lastterm: 6]
I161111 17:23:19.543133 279 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 35 [logterm: 0, index: 395] rejected msgApp [logterm: 6, index: 395] from 1
I161111 17:23:19.543339 93 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 393) from 35 for index 395
I161111 17:23:19.543366 93 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 35 to [next = 394, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.563131 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:53 > next_replica_id:54
I161111 17:23:19.565996 2967 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:53}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:19.569360 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7b9ff5f5 at index 400 in 52.961µs.
I161111 17:23:19.592250 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 519, log entries: 62
I161111 17:23:19.593616 452 storage/store.go:2988  [s3,r1/53:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:19.598325 340 storage/store.go:2041  [s3,r1/53:/M{in-ax}] DEL 1
I161111 17:23:19.599679 2842 storage/store.go:1912  [s3] ADD 1
I161111 17:23:19.599785 2842 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:19.601483 2842 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:19.601530 2842 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:19.601575 2842 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:19.601610 2842 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:19.602789 2842 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 400, term: 6]
I161111 17:23:19.602841 2842 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 400, term: 6]
I161111 17:23:19.602892 2842 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 401, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.602936 2842 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 401, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:19.602970 2842 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 400] restored snapshot [index: 400, term: 6]
I161111 17:23:19.603014 2842 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 400 (id=7b9ff5f5, encoded size=16, 1 rocksdb batches, 62 log entries)
I161111 17:23:19.954294 2842 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.351s
I161111 17:23:19.954335 2842 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:19.955511 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:54
I161111 17:23:19.957114 2999 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:54}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:54}]
I161111 17:23:20.053935 247 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 36 became follower at term 6
I161111 17:23:20.053983 247 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 36 [peers: [1,2], term: 6, commit: 400, applied: 400, lastindex: 400, lastterm: 6]
I161111 17:23:20.079315 249 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 36 [logterm: 0, index: 408] rejected msgApp [logterm: 6, index: 408] from 1
I161111 17:23:20.079556 69 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 400) from 36 for index 408
I161111 17:23:20.079583 69 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 36 to [next = 401, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.105086 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:54 > next_replica_id:55
I161111 17:23:20.136446 3016 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:54}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:20.139114 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1461b8eb at index 413 in 51.887µs.
I161111 17:23:20.145764 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 534, log entries: 75
I161111 17:23:20.146344 452 storage/store.go:2988  [s3,r1/54:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:20.165349 340 storage/store.go:2041  [s3,r1/54:/M{in-ax}] DEL 1
I161111 17:23:20.165844 3049 storage/store.go:1912  [s3] ADD 1
I161111 17:23:20.165957 3049 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:20.183134 3049 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:20.183180 3049 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:20.183206 3049 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:20.183219 3049 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:20.193279 3049 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 413, term: 6]
I161111 17:23:20.217850 3049 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 413, term: 6]
I161111 17:23:20.217910 3049 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 414, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.217950 3049 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 414, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.217972 3049 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 413] restored snapshot [index: 413, term: 6]
I161111 17:23:20.218017 3049 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 413 (id=1461b8eb, encoded size=16, 1 rocksdb batches, 75 log entries)
I161111 17:23:20.310646 3049 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.093s
I161111 17:23:20.313902 3049 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:20.314656 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:55
I161111 17:23:20.316109 3031 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:55}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:55}]
I161111 17:23:20.438388 270 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 37 became follower at term 6
I161111 17:23:20.438433 270 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 37 [peers: [1,2], term: 6, commit: 413, applied: 413, lastindex: 413, lastterm: 6]
I161111 17:23:20.438481 270 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 37 [logterm: 0, index: 418] rejected msgApp [logterm: 6, index: 418] from 1
I161111 17:23:20.438685 77 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 413) from 37 for index 418
I161111 17:23:20.438714 77 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 37 to [next = 414, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.618823 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:55 > next_replica_id:56
I161111 17:23:20.621886 3093 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:55}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:20.628696 452 storage/store.go:2988  [s3,r1/55:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:20.636326 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 41220a5d at index 424 in 165.54µs.
I161111 17:23:20.638929 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 547, log entries: 86
I161111 17:23:20.648857 340 storage/store.go:2041  [s3,r1/55:/M{in-ax}] DEL 1
I161111 17:23:20.649126 3112 storage/store.go:1912  [s3] ADD 1
I161111 17:23:20.649207 3112 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:20.650834 3112 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:20.650868 3112 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:20.650903 3112 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:20.650926 3112 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:20.652081 3112 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 424, term: 6]
I161111 17:23:20.657164 3112 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 424, term: 6]
I161111 17:23:20.657202 3112 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 425, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.657231 3112 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 425, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.657253 3112 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 424] restored snapshot [index: 424, term: 6]
I161111 17:23:20.657296 3112 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 424 (id=41220a5d, encoded size=16, 1 rocksdb batches, 86 log entries)
I161111 17:23:20.827068 3112 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.170s
I161111 17:23:20.827251 3112 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:20.828598 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:56
I161111 17:23:20.830129 3155 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:56}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:56}]
I161111 17:23:20.908871 286 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 38 became follower at term 6
I161111 17:23:20.908922 286 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 38 [peers: [1,2], term: 6, commit: 424, applied: 424, lastindex: 424, lastterm: 6]
I161111 17:23:20.937466 241 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 38 [logterm: 0, index: 431] rejected msgApp [logterm: 6, index: 431] from 1
I161111 17:23:20.937645 107 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 424) from 38 for index 431
I161111 17:23:20.937672 107 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 38 to [next = 425, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:20.976361 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:56 > next_replica_id:57
I161111 17:23:20.978318 3103 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:56}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:20.989328 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 73036b35 at index 433 in 57.514µs.
I161111 17:23:20.990415 452 storage/store.go:2988  [s3,r1/56:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:21.034158 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 558, log entries: 95
I161111 17:23:21.037920 340 storage/store.go:2041  [s3,r1/56:/M{in-ax}] DEL 1
I161111 17:23:21.038295 3163 storage/store.go:1912  [s3] ADD 1
I161111 17:23:21.038374 3163 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:21.051836 3163 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:21.052538 3163 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:21.052576 3163 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:21.052598 3163 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:21.057675 3163 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 433, term: 6]
I161111 17:23:21.069426 3163 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 433, term: 6]
I161111 17:23:21.069472 3163 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 434, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.069506 3163 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 434, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.072034 3163 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 433] restored snapshot [index: 433, term: 6]
I161111 17:23:21.072073 3163 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 433 (id=73036b35, encoded size=16, 1 rocksdb batches, 95 log entries)
I161111 17:23:21.145182 3163 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.073s
I161111 17:23:21.145418 3163 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:21.146462 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:57
I161111 17:23:21.148042 3130 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:57}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:57}]
I161111 17:23:21.173421 253 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 39 became follower at term 6
I161111 17:23:21.173467 253 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 39 [peers: [1,2], term: 6, commit: 433, applied: 433, lastindex: 433, lastterm: 6]
I161111 17:23:21.173520 253 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 39 [logterm: 0, index: 438] rejected msgApp [logterm: 6, index: 438] from 1
I161111 17:23:21.173697 103 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 433) from 39 for index 438
I161111 17:23:21.173724 103 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 39 to [next = 434, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.192885 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:57 > next_replica_id:58
I161111 17:23:21.194514 3148 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:57}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:21.197187 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 339-443
I161111 17:23:21.204662 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ac9aa07e at index 443 in 64.65µs.
I161111 17:23:21.206458 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 570, log entries: 105
I161111 17:23:21.220157 3153 raft/raft.go:568  [s3,r1/57:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:21.220237 3153 raft/raft.go:316  [s3,r1/57:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 443, applied: 443, lastindex: 443, lastterm: 6]
I161111 17:23:21.220320 3153 raft/raft.go:1106  [s3,r1/57:/M{in-ax}] ffffffffffffffff [commit: 443] ignored snapshot [index: 443, term: 6]
I161111 17:23:21.222613 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:58
I161111 17:23:21.241869 452 storage/store.go:2988  [s3,r1/57:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:21.249799 3132 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:58}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:58}]
I161111 17:23:21.298880 340 storage/store.go:2041  [s3,r1/57:/M{in-ax}] DEL 1
I161111 17:23:21.338435 264 raft/raft.go:568  [s3,r1/58:{-}] 3a became follower at term 0
I161111 17:23:21.340261 264 raft/raft.go:316  [s3,r1/58:{-}] newRaft 3a [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:21.340278 264 raft/raft.go:568  [s3,r1/58:{-}] 3a became follower at term 1
I161111 17:23:21.340300 264 raft/raft.go:710  [s3,r1/58:{-}] 3a [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:21.340311 264 raft/raft.go:568  [s3,r1/58:{-}] 3a became follower at term 6
I161111 17:23:21.340326 264 raft/raft.go:1088  [s3,r1/58:{-}] 3a [logterm: 0, index: 446] rejected msgApp [logterm: 6, index: 446] from 1
I161111 17:23:21.340559 96 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 3a for index 446
I161111 17:23:21.340585 96 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3a to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.340695 96 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8abced34 at index 446 in 63.577µs.
I161111 17:23:21.340728 96 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 443, commit: 446] sent snapshot[index: 446, term: 6] to 3a [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.340744 96 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 3a [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 446]
I161111 17:23:21.344572 3212 storage/store.go:1912  [s3] ADD 1
I161111 17:23:21.346389 3212 raft/raft.go:1125  [s3,r1/58:{-}] 3a [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 446, term: 6]
I161111 17:23:21.346411 3212 raft/log.go:288  [s3,r1/58:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 446, term: 6]
I161111 17:23:21.346431 3212 raft/raft.go:1135  [s3,r1/58:{-}] 3a restored progress of 1 [next = 447, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.346446 3212 raft/raft.go:1135  [s3,r1/58:{-}] 3a restored progress of 2 [next = 447, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.346472 3212 raft/raft.go:1135  [s3,r1/58:{-}] 3a restored progress of 3a [next = 447, match = 446, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.346486 3212 raft/raft.go:1102  [s3,r1/58:{-}] 3a [commit: 446] restored snapshot [index: 446, term: 6]
I161111 17:23:21.346512 3212 storage/replica_raftstorage.go:589  [s3,r1/58:{-}] applying Raft snapshot at index 446 (id=8abced34, encoded size=16, 1 rocksdb batches, 4 log entries)
I161111 17:23:21.350547 3133 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 573, log entries: 4
I161111 17:23:21.353084 3212 storage/replica_raftstorage.go:592  [s3,r1/58:/M{in-ax}] applied Raft snapshot in 0.007s
I161111 17:23:21.353708 100 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 3a [next = 447, match = 446, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 446]
I161111 17:23:21.359907 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:58 > next_replica_id:59
I161111 17:23:21.361904 3276 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:58}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:21.399097 452 storage/store.go:2988  [s3,r1/58:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:21.442361 3133 raft/raft.go:854  [s1,r1/1:/M{in-ax}] 1 no progress available for 3a
I161111 17:23:21.442555 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 138dfa68 at index 451 in 85.876µs.
I161111 17:23:21.443951 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 579, log entries: 9
I161111 17:23:21.530860 340 storage/store.go:2041  [s3,r1/58:/M{in-ax}] DEL 1
I161111 17:23:21.590156 3216 storage/store.go:1912  [s3] ADD 1
I161111 17:23:21.590221 3216 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:21.592095 3216 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:21.592129 3216 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:21.592152 3216 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:21.592165 3216 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:21.593367 3216 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 451, term: 6]
I161111 17:23:21.593386 3216 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 451, term: 6]
I161111 17:23:21.593404 3216 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 452, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.593421 3216 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 452, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.593433 3216 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 451] restored snapshot [index: 451, term: 6]
I161111 17:23:21.593450 3216 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 451 (id=138dfa68, encoded size=16, 1 rocksdb batches, 9 log entries)
I161111 17:23:21.618948 3216 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.025s
I161111 17:23:21.619006 3216 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:21.619704 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:59
I161111 17:23:21.621260 3239 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:59}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:59}]
I161111 17:23:21.699716 279 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3b became follower at term 6
I161111 17:23:21.699760 279 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3b [peers: [1,2], term: 6, commit: 451, applied: 451, lastindex: 451, lastterm: 6]
I161111 17:23:21.699802 279 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3b [logterm: 0, index: 456] rejected msgApp [logterm: 6, index: 456] from 1
I161111 17:23:21.700912 103 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 451) from 3b for index 456
I161111 17:23:21.700936 103 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3b to [next = 452, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.713731 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:59 > next_replica_id:60
I161111 17:23:21.717412 3317 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:59}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:21.719805 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 406f864c at index 460 in 50.257µs.
I161111 17:23:21.725459 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 590, log entries: 18
I161111 17:23:21.754998 340 storage/store.go:2041  [s3,r1/59:/M{in-ax}] DEL 1
I161111 17:23:21.775142 3181 storage/store.go:1912  [s3] ADD 1
I161111 17:23:21.775282 3181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:21.778187 3181 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:21.778218 3181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:21.778275 3181 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:21.778300 3181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
E161111 17:23:21.781365 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:21.781854 3181 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 460, term: 6]
I161111 17:23:21.781888 3181 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 460, term: 6]
I161111 17:23:21.781919 3181 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 461, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.782219 3181 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 461, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.782245 3181 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 460] restored snapshot [index: 460, term: 6]
I161111 17:23:21.782272 3181 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 460 (id=406f864c, encoded size=16, 1 rocksdb batches, 18 log entries)
I161111 17:23:21.795792 3181 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.013s
I161111 17:23:21.802164 3181 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:21.802453 243 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3b became follower at term 6
I161111 17:23:21.802513 243 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3b [peers: [1,2], term: 6, commit: 460, applied: 460, lastindex: 460, lastterm: 6]
I161111 17:23:21.803292 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:60
I161111 17:23:21.804836 3185 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:60}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:60}]
I161111 17:23:21.808099 245 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3c became follower at term 6
I161111 17:23:21.808153 245 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3c [peers: [1,2], term: 6, commit: 460, applied: 460, lastindex: 460, lastterm: 6]
I161111 17:23:21.837524 247 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3c [logterm: 0, index: 463] rejected msgApp [logterm: 6, index: 463] from 1
I161111 17:23:21.837670 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:21.838270 96 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 460) from 3c for index 463
I161111 17:23:21.838296 96 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3c to [next = 461, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:21.846255 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:60 > next_replica_id:61
I161111 17:23:21.878338 3254 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:60}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:21.924666 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot f92b7bcc at index 465 in 65.363µs.
I161111 17:23:21.926048 452 storage/store.go:2988  [s3,r1/60:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:21.927377 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 597, log entries: 23
I161111 17:23:22.031068 340 storage/store.go:2041  [s3,r1/60:/M{in-ax}] DEL 1
I161111 17:23:22.040294 3303 storage/store.go:1912  [s3] ADD 1
I161111 17:23:22.040947 3303 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:22.042885 3303 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:22.043617 3303 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:22.043658 3303 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:22.043685 3303 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:22.069425 3303 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 465, term: 6]
I161111 17:23:22.069469 3303 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 465, term: 6]
I161111 17:23:22.070033 3303 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 466, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.070071 3303 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 466, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.070117 3303 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 465] restored snapshot [index: 465, term: 6]
I161111 17:23:22.070157 3303 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 465 (id=f92b7bcc, encoded size=16, 1 rocksdb batches, 23 log entries)
I161111 17:23:22.104618 3303 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.034s
I161111 17:23:22.104673 3303 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:22.105591 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:61
I161111 17:23:22.107237 3373 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:61}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:61}]
I161111 17:23:22.203826 260 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3d became follower at term 6
I161111 17:23:22.203878 260 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3d [peers: [1,2], term: 6, commit: 465, applied: 465, lastindex: 465, lastterm: 6]
I161111 17:23:22.239911 267 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3d [logterm: 0, index: 473] rejected msgApp [logterm: 6, index: 473] from 1
I161111 17:23:22.241101 106 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 465) from 3d for index 473
I161111 17:23:22.241133 106 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3d to [next = 466, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.254526 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:61 > next_replica_id:62
I161111 17:23:22.268342 3375 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:61}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:22.270392 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 70a22b1f at index 478 in 66.995µs.
I161111 17:23:22.294404 452 storage/store.go:2988  [s3,r1/61:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:22.298970 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 612, log entries: 36
I161111 17:23:22.375104 340 storage/store.go:2041  [s3,r1/61:/M{in-ax}] DEL 1
I161111 17:23:22.380384 3430 storage/store.go:1912  [s3] ADD 1
I161111 17:23:22.381427 3430 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:22.415932 3430 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:22.416090 3430 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:22.416165 3430 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:22.416249 3430 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:22.460570 3430 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 478, term: 6]
I161111 17:23:22.460650 3430 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 478, term: 6]
I161111 17:23:22.460820 3430 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 479, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.461065 3430 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 479, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.461138 3430 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 478] restored snapshot [index: 478, term: 6]
I161111 17:23:22.478266 3430 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 478 (id=70a22b1f, encoded size=16, 1 rocksdb batches, 36 log entries)
I161111 17:23:22.650420 3430 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.172s
I161111 17:23:22.653341 3430 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:22.660755 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:62
I161111 17:23:22.662380 3493 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:62}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:62}]
I161111 17:23:22.753980 243 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3e became follower at term 6
I161111 17:23:22.754034 243 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3e [peers: [1,2], term: 6, commit: 478, applied: 478, lastindex: 478, lastterm: 6]
I161111 17:23:22.780356 245 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3e [logterm: 0, index: 486] rejected msgApp [logterm: 6, index: 486] from 1
I161111 17:23:22.780554 102 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 478) from 3e for index 486
I161111 17:23:22.780589 102 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3e to [next = 479, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.809070 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:62 > next_replica_id:63
I161111 17:23:22.827145 3498 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:62}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:22.837683 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 068e6232 at index 491 in 64.671µs.
I161111 17:23:22.839155 452 storage/store.go:2988  [s3,r1/62:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:22.840848 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 627, log entries: 49
I161111 17:23:22.841293 452 storage/store.go:2988  [s3,r1/62:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:22.860421 340 storage/store.go:2041  [s3,r1/62:/M{in-ax}] DEL 1
I161111 17:23:22.870118 3464 storage/store.go:1912  [s3] ADD 1
I161111 17:23:22.871044 3464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:22.894494 3464 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:22.894762 3464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:22.894927 3464 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:22.895078 3464 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:22.947857 3464 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 491, term: 6]
I161111 17:23:22.948008 3464 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 491, term: 6]
I161111 17:23:22.948040 3464 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 492, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.948336 3464 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 492, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:22.948361 3464 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 491] restored snapshot [index: 491, term: 6]
I161111 17:23:22.948502 3464 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 491 (id=068e6232, encoded size=16, 1 rocksdb batches, 49 log entries)
I161111 17:23:23.062227 3464 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.114s
I161111 17:23:23.062374 3464 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:23.063313 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:63
I161111 17:23:23.064804 3519 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:63}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:63}]
I161111 17:23:23.156297 270 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3f became follower at term 6
I161111 17:23:23.156346 270 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3f [peers: [1,2], term: 6, commit: 491, applied: 491, lastindex: 491, lastterm: 6]
I161111 17:23:23.156411 270 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 3f [logterm: 0, index: 496] rejected msgApp [logterm: 6, index: 496] from 1
I161111 17:23:23.156587 81 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 491) from 3f for index 496
I161111 17:23:23.156614 81 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 3f to [next = 492, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.230917 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:63 > next_replica_id:64
I161111 17:23:23.235491 3543 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:63}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:23.237872 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot beeea332 at index 501 in 62.413µs.
I161111 17:23:23.239745 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 639, log entries: 59
I161111 17:23:23.240239 452 storage/store.go:2988  [s3,r1/63:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:23.246602 340 storage/store.go:2041  [s3,r1/63:/M{in-ax}] DEL 1
I161111 17:23:23.249332 3577 storage/store.go:1912  [s3] ADD 1
I161111 17:23:23.249560 3577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:23.255016 3577 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:23.255233 3577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:23.255414 3577 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:23.255555 3577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:23.349178 3577 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 501, term: 6]
I161111 17:23:23.349888 3577 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 501, term: 6]
I161111 17:23:23.429529 3577 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 502, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.429581 3577 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 502, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.429602 3577 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 501] restored snapshot [index: 501, term: 6]
I161111 17:23:23.429640 3577 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 501 (id=beeea332, encoded size=16, 1 rocksdb batches, 59 log entries)
I161111 17:23:23.439034 3577 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.009s
I161111 17:23:23.439093 3577 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:23.439309 282 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 3f became follower at term 6
I161111 17:23:23.439382 282 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 3f [peers: [1,2], term: 6, commit: 501, applied: 501, lastindex: 501, lastterm: 6]
I161111 17:23:23.440099 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:64
I161111 17:23:23.442203 3621 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:64}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:64}]
I161111 17:23:23.496391 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:23.511352 287 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 40 became follower at term 6
I161111 17:23:23.511398 287 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 40 [peers: [1,2], term: 6, commit: 501, applied: 501, lastindex: 501, lastterm: 6]
I161111 17:23:23.511444 287 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 40 [logterm: 0, index: 506] rejected msgApp [logterm: 6, index: 506] from 1
I161111 17:23:23.511646 93 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 501) from 40 for index 506
I161111 17:23:23.511671 93 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 40 to [next = 502, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.528069 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:64 > next_replica_id:65
I161111 17:23:23.530205 3627 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:64}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:23.532873 452 storage/store.go:2988  [s3,r1/64:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:23.557281 340 storage/store.go:2041  [s3,r1/64:/M{in-ax}] DEL 1
I161111 17:23:23.597685 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9b1e3279 at index 511 in 67.33µs.
W161111 17:23:23.600656 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 64}: raft group deleted
I161111 17:23:23.601857 3630 storage/store.go:1912  [s3] ADD 1
I161111 17:23:23.601928 3630 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:23.603959 3630 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:23.603981 3630 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:23.604007 3630 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:23.604021 3630 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:23.605540 3630 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 511, term: 6]
I161111 17:23:23.605565 3630 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 511, term: 6]
I161111 17:23:23.605589 3630 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 512, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.605605 3630 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 512, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.605618 3630 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 511] restored snapshot [index: 511, term: 6]
I161111 17:23:23.605637 3630 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 511 (id=9b1e3279, encoded size=16, 1 rocksdb batches, 69 log entries)
I161111 17:23:23.608176 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 651, log entries: 69
I161111 17:23:23.680575 3630 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.075s
I161111 17:23:23.680623 3630 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:23.687176 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:65
I161111 17:23:23.689181 3598 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:65}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:65}]
I161111 17:23:23.696397 255 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 41 became follower at term 6
I161111 17:23:23.696454 255 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 41 [peers: [1,2], term: 6, commit: 511, applied: 511, lastindex: 511, lastterm: 6]
I161111 17:23:23.696547 255 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 41 [logterm: 0, index: 514] rejected msgApp [logterm: 6, index: 514] from 1
I161111 17:23:23.696732 90 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 511) from 41 for index 514
I161111 17:23:23.696758 90 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 41 to [next = 512, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.702088 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:65 > next_replica_id:66
I161111 17:23:23.704349 3683 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:65}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:23.707233 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ca15c0d4 at index 518 in 62.595µs.
I161111 17:23:23.710388 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 660, log entries: 76
I161111 17:23:23.717146 452 storage/store.go:2988  [s3,r1/65:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:23.721428 3699 raft/raft.go:568  [s3,r1/65:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:23.721523 3699 raft/raft.go:316  [s3,r1/65:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 518, applied: 518, lastindex: 518, lastterm: 6]
I161111 17:23:23.721583 3699 raft/raft.go:1106  [s3,r1/65:/M{in-ax}] ffffffffffffffff [commit: 518] ignored snapshot [index: 518, term: 6]
I161111 17:23:23.724040 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:66
I161111 17:23:23.726448 3670 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:66}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:66}]
I161111 17:23:23.740689 340 storage/store.go:2041  [s3,r1/65:/M{in-ax}] DEL 1
I161111 17:23:23.857154 263 raft/raft.go:568  [s3,r1/66:{-}] 42 became follower at term 0
I161111 17:23:23.859262 263 raft/raft.go:316  [s3,r1/66:{-}] newRaft 42 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:23.859281 263 raft/raft.go:568  [s3,r1/66:{-}] 42 became follower at term 1
I161111 17:23:23.859303 263 raft/raft.go:710  [s3,r1/66:{-}] 42 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:23.859314 263 raft/raft.go:568  [s3,r1/66:{-}] 42 became follower at term 6
I161111 17:23:23.861165 260 raft/raft.go:1088  [s3,r1/66:{-}] 42 [logterm: 0, index: 520] rejected msgApp [logterm: 6, index: 520] from 1
I161111 17:23:23.861339 76 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 42 for index 520
I161111 17:23:23.861363 76 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 42 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.861489 76 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 062c2820 at index 520 in 77.754µs.
I161111 17:23:23.861529 76 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 443, commit: 520] sent snapshot[index: 520, term: 6] to 42 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.861547 76 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 42 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 520]
I161111 17:23:23.929695 3549 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 663, log entries: 78
I161111 17:23:23.929829 3601 storage/store.go:1912  [s3] ADD 1
I161111 17:23:23.931922 3601 raft/raft.go:1125  [s3,r1/66:{-}] 42 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 520, term: 6]
I161111 17:23:23.931946 3601 raft/log.go:288  [s3,r1/66:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 520, term: 6]
I161111 17:23:23.931968 3601 raft/raft.go:1135  [s3,r1/66:{-}] 42 restored progress of 1 [next = 521, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.931984 3601 raft/raft.go:1135  [s3,r1/66:{-}] 42 restored progress of 2 [next = 521, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.932000 3601 raft/raft.go:1135  [s3,r1/66:{-}] 42 restored progress of 42 [next = 521, match = 520, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:23.932013 3601 raft/raft.go:1102  [s3,r1/66:{-}] 42 [commit: 520] restored snapshot [index: 520, term: 6]
I161111 17:23:23.932040 3601 storage/replica_raftstorage.go:589  [s3,r1/66:{-}] applying Raft snapshot at index 520 (id=062c2820, encoded size=16, 1 rocksdb batches, 78 log entries)
I161111 17:23:23.983851 3601 storage/replica_raftstorage.go:592  [s3,r1/66:/M{in-ax}] applied Raft snapshot in 0.052s
I161111 17:23:23.984464 96 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 42 [next = 521, match = 520, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 520]
I161111 17:23:24.019498 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:66 > next_replica_id:67
I161111 17:23:24.021534 3726 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:66}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:24.026870 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 339ef961 at index 525 in 2.766704ms.
I161111 17:23:24.029577 452 storage/store.go:2988  [s3,r1/66:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:24.031497 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 669, log entries: 83
I161111 17:23:24.134219 340 storage/store.go:2041  [s3,r1/66:/M{in-ax}] DEL 1
I161111 17:23:24.134988 3737 storage/store.go:1912  [s3] ADD 1
I161111 17:23:24.135052 3737 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:24.137223 3737 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:24.137242 3737 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:24.137264 3737 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:24.137276 3737 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:24.138894 3737 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 525, term: 6]
I161111 17:23:24.138917 3737 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 525, term: 6]
I161111 17:23:24.138938 3737 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 526, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.138953 3737 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 526, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.138966 3737 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 525] restored snapshot [index: 525, term: 6]
I161111 17:23:24.138990 3737 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 525 (id=339ef961, encoded size=16, 1 rocksdb batches, 83 log entries)
I161111 17:23:24.169291 3737 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.030s
I161111 17:23:24.170874 3737 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:24.171149 277 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 42 became follower at term 6
I161111 17:23:24.171194 277 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 42 [peers: [1,2], term: 6, commit: 525, applied: 525, lastindex: 525, lastterm: 6]
I161111 17:23:24.171779 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:67
I161111 17:23:24.173730 3765 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:67}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:67}]
I161111 17:23:24.191496 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:24.221361 282 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 43 became follower at term 6
I161111 17:23:24.221409 282 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 43 [peers: [1,2], term: 6, commit: 525, applied: 525, lastindex: 525, lastterm: 6]
I161111 17:23:24.221456 282 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 43 [logterm: 0, index: 530] rejected msgApp [logterm: 6, index: 530] from 1
I161111 17:23:24.221617 89 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 525) from 43 for index 530
I161111 17:23:24.221641 89 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 43 to [next = 526, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.253308 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:67 > next_replica_id:68
I161111 17:23:24.257995 3812 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:67}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:24.311631 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0b64f38d at index 535 in 83.918µs.
I161111 17:23:24.318611 452 storage/store.go:2988  [s3,r1/67:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:24.327757 452 storage/store.go:2988  [s3,r1/67:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:24.333584 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 681, log entries: 93
I161111 17:23:24.336254 340 storage/store.go:2041  [s3,r1/67:/M{in-ax}] DEL 1
I161111 17:23:24.336536 3775 storage/store.go:1912  [s3] ADD 1
I161111 17:23:24.336659 3775 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
E161111 17:23:24.337344 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:24.342104 3775 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:24.400093 3775 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:24.400253 3775 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:24.400283 3775 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:24.444493 3775 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 535, term: 6]
I161111 17:23:24.444548 3775 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 535, term: 6]
I161111 17:23:24.444599 3775 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 536, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.444647 3775 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 536, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.444676 3775 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 535] restored snapshot [index: 535, term: 6]
I161111 17:23:24.444726 3775 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 535 (id=0b64f38d, encoded size=16, 1 rocksdb batches, 93 log entries)
I161111 17:23:24.711161 3775 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.266s
I161111 17:23:24.711219 3775 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:24.711987 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:68
I161111 17:23:24.713620 3865 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:68}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:68}]
I161111 17:23:24.758556 247 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 44 became follower at term 6
I161111 17:23:24.758602 247 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 44 [peers: [1,2], term: 6, commit: 535, applied: 535, lastindex: 535, lastterm: 6]
I161111 17:23:24.758648 247 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 44 [logterm: 0, index: 543] rejected msgApp [logterm: 6, index: 543] from 1
I161111 17:23:24.758819 97 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 535) from 44 for index 543
I161111 17:23:24.758846 97 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 44 to [next = 536, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.810530 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:68 > next_replica_id:69
I161111 17:23:24.813414 3883 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:68}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:24.815081 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 443-547
I161111 17:23:24.817747 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 6295f19c at index 548 in 60.12µs.
I161111 17:23:24.825776 452 storage/store.go:2988  [s3,r1/68:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:24.877119 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 696, log entries: 106
I161111 17:23:24.888337 340 storage/store.go:2041  [s3,r1/68:/M{in-ax}] DEL 1
I161111 17:23:24.889222 3787 storage/store.go:1912  [s3] ADD 1
I161111 17:23:24.889407 3787 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:24.931864 3787 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:24.931922 3787 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:24.931956 3787 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:24.932002 3787 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:24.984144 3787 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 548, term: 6]
I161111 17:23:24.984258 3787 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 548, term: 6]
I161111 17:23:24.984508 3787 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 549, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.984546 3787 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 549, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:24.984792 3787 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 548] restored snapshot [index: 548, term: 6]
I161111 17:23:24.984900 3787 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 548 (id=6295f19c, encoded size=16, 1 rocksdb batches, 106 log entries)
I161111 17:23:25.045515 3787 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.060s
I161111 17:23:25.057772 3787 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:25.058987 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:69
I161111 17:23:25.060989 3681 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:69}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:69}]
I161111 17:23:25.105260 265 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 45 became follower at term 6
I161111 17:23:25.105306 265 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 45 [peers: [1,2], term: 6, commit: 548, applied: 548, lastindex: 548, lastterm: 6]
I161111 17:23:25.119596 273 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 45 [logterm: 0, index: 554] rejected msgApp [logterm: 6, index: 554] from 1
I161111 17:23:25.120672 81 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 548) from 45 for index 554
I161111 17:23:25.120697 81 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 45 to [next = 549, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.352466 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:69 > next_replica_id:70
I161111 17:23:25.354185 3928 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:69}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:25.358445 452 storage/store.go:2988  [s3,r1/69:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:25.362773 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a3640e65 at index 562 in 89.57µs.
I161111 17:23:25.364515 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 711, log entries: 16
I161111 17:23:25.389758 340 storage/store.go:2041  [s3,r1/69:/M{in-ax}] DEL 1
I161111 17:23:25.390357 3933 storage/store.go:1912  [s3] ADD 1
I161111 17:23:25.390481 3933 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:25.396042 3933 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:25.396099 3933 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:25.396161 3933 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:25.396193 3933 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:25.398736 3933 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 562, term: 6]
I161111 17:23:25.398775 3933 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 562, term: 6]
I161111 17:23:25.398811 3933 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 563, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.398839 3933 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 563, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.398864 3933 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 562] restored snapshot [index: 562, term: 6]
I161111 17:23:25.398895 3933 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 562 (id=a3640e65, encoded size=16, 1 rocksdb batches, 16 log entries)
I161111 17:23:25.572806 3933 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.174s
I161111 17:23:25.572868 3933 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:25.573659 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:70
I161111 17:23:25.575174 3937 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:70}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:70}]
I161111 17:23:25.656469 247 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 46 became follower at term 6
I161111 17:23:25.656519 247 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 46 [peers: [1,2], term: 6, commit: 562, applied: 562, lastindex: 562, lastterm: 6]
I161111 17:23:25.659097 254 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 46 [logterm: 0, index: 567] rejected msgApp [logterm: 6, index: 567] from 1
I161111 17:23:25.659276 71 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 562) from 46 for index 567
I161111 17:23:25.659303 71 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 46 to [next = 563, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.717677 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:70 > next_replica_id:71
I161111 17:23:25.719585 4029 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:70}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:25.722051 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 8562103d at index 572 in 61.38µs.
I161111 17:23:25.726238 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 723, log entries: 26
I161111 17:23:25.726381 452 storage/store.go:2988  [s3,r1/70:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:25.726586 3988 raft/raft.go:568  [s3,r1/70:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:25.726620 3988 raft/raft.go:316  [s3,r1/70:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 572, applied: 572, lastindex: 572, lastterm: 6]
I161111 17:23:25.726668 3988 raft/raft.go:1106  [s3,r1/70:/M{in-ax}] ffffffffffffffff [commit: 572] ignored snapshot [index: 572, term: 6]
I161111 17:23:25.733091 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:71
I161111 17:23:25.734750 3976 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:71}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:71}]
I161111 17:23:25.807797 340 storage/store.go:2041  [s3,r1/70:/M{in-ax}] DEL 1
W161111 17:23:25.830641 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 70}: raft group deleted
I161111 17:23:25.841881 271 raft/raft.go:568  [s3,r1/71:{-}] 47 became follower at term 0
I161111 17:23:25.844220 271 raft/raft.go:316  [s3,r1/71:{-}] newRaft 47 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:25.844242 271 raft/raft.go:568  [s3,r1/71:{-}] 47 became follower at term 1
I161111 17:23:25.844266 271 raft/raft.go:710  [s3,r1/71:{-}] 47 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:25.844278 271 raft/raft.go:568  [s3,r1/71:{-}] 47 became follower at term 6
I161111 17:23:25.844297 271 raft/raft.go:1088  [s3,r1/71:{-}] 47 [logterm: 0, index: 574] rejected msgApp [logterm: 6, index: 574] from 1
I161111 17:23:25.846220 79 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 47 for index 574
I161111 17:23:25.846251 79 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 47 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.846351 79 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 957e34ac at index 575 in 56.943µs.
I161111 17:23:25.846386 79 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 547, commit: 575] sent snapshot[index: 575, term: 6] to 47 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.846401 79 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 47 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 575]
I161111 17:23:25.849236 3964 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 727, log entries: 29
I161111 17:23:25.850799 4052 storage/store.go:1912  [s3] ADD 1
I161111 17:23:25.873452 4052 raft/raft.go:1125  [s3,r1/71:{-}] 47 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 575, term: 6]
I161111 17:23:25.873513 4052 raft/log.go:288  [s3,r1/71:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 575, term: 6]
I161111 17:23:25.873550 4052 raft/raft.go:1135  [s3,r1/71:{-}] 47 restored progress of 1 [next = 576, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.873582 4052 raft/raft.go:1135  [s3,r1/71:{-}] 47 restored progress of 2 [next = 576, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.873612 4052 raft/raft.go:1135  [s3,r1/71:{-}] 47 restored progress of 47 [next = 576, match = 575, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:25.873647 4052 raft/raft.go:1102  [s3,r1/71:{-}] 47 [commit: 575] restored snapshot [index: 575, term: 6]
I161111 17:23:25.873706 4052 storage/replica_raftstorage.go:589  [s3,r1/71:{-}] applying Raft snapshot at index 575 (id=957e34ac, encoded size=16, 1 rocksdb batches, 29 log entries)
I161111 17:23:25.922189 4052 storage/replica_raftstorage.go:592  [s3,r1/71:/M{in-ax}] applied Raft snapshot in 0.048s
I161111 17:23:25.922538 83 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 47 [next = 576, match = 575, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 575]
I161111 17:23:25.949203 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:71 > next_replica_id:72
I161111 17:23:25.984733 3989 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:71}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:26.004450 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9e0bea0d at index 579 in 79.183µs.
I161111 17:23:26.023055 340 storage/store.go:2041  [s3,r1/71:/M{in-ax}] DEL 1
I161111 17:23:26.025348 4053 storage/store.go:1912  [s3] ADD 1
I161111 17:23:26.025412 4053 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:26.027812 4053 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:26.027831 4053 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:26.027853 4053 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:26.027865 4053 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:26.029833 4053 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 579, term: 6]
I161111 17:23:26.029870 4053 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 579, term: 6]
I161111 17:23:26.029903 4053 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 580, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.029930 4053 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 580, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.029952 4053 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 579] restored snapshot [index: 579, term: 6]
I161111 17:23:26.029986 4053 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 579 (id=9e0bea0d, encoded size=16, 1 rocksdb batches, 33 log entries)
I161111 17:23:26.047851 4053 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.018s
I161111 17:23:26.051486 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 732, log entries: 33
I161111 17:23:26.053003 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:26.088333 4053 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:26.101753 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:72
I161111 17:23:26.103481 4102 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:72}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:72}]
I161111 17:23:26.247614 340 storage/store.go:2041  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:26.280073 283 raft/raft.go:568  [s3,r1/72:{-}] 48 became follower at term 0
I161111 17:23:26.282502 283 raft/raft.go:316  [s3,r1/72:{-}] newRaft 48 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:26.282534 283 raft/raft.go:568  [s3,r1/72:{-}] 48 became follower at term 1
I161111 17:23:26.282559 283 raft/raft.go:710  [s3,r1/72:{-}] 48 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:26.282572 283 raft/raft.go:568  [s3,r1/72:{-}] 48 became follower at term 6
I161111 17:23:26.381015 245 raft/raft.go:1088  [s3,r1/72:{-}] 48 [logterm: 0, index: 584] rejected msgApp [logterm: 6, index: 584] from 1
I161111 17:23:26.382106 104 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 48 for index 584
I161111 17:23:26.382142 104 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 48 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.382251 104 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ee7fabf4 at index 588 in 60.795µs.
I161111 17:23:26.382292 104 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 547, commit: 588] sent snapshot[index: 588, term: 6] to 48 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.382313 104 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 48 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 588]
I161111 17:23:26.391468 4120 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 742, log entries: 42
I161111 17:23:26.391645 4125 storage/store.go:1912  [s3] ADD 1
I161111 17:23:26.394112 4125 raft/raft.go:1125  [s3,r1/72:{-}] 48 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 588, term: 6]
I161111 17:23:26.394133 4125 raft/log.go:288  [s3,r1/72:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 588, term: 6]
I161111 17:23:26.394154 4125 raft/raft.go:1135  [s3,r1/72:{-}] 48 restored progress of 1 [next = 589, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.394170 4125 raft/raft.go:1135  [s3,r1/72:{-}] 48 restored progress of 2 [next = 589, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.394186 4125 raft/raft.go:1135  [s3,r1/72:{-}] 48 restored progress of 48 [next = 589, match = 588, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.394200 4125 raft/raft.go:1102  [s3,r1/72:{-}] 48 [commit: 588] restored snapshot [index: 588, term: 6]
I161111 17:23:26.394228 4125 storage/replica_raftstorage.go:589  [s3,r1/72:{-}] applying Raft snapshot at index 588 (id=ee7fabf4, encoded size=16, 1 rocksdb batches, 42 log entries)
I161111 17:23:26.417468 4125 storage/replica_raftstorage.go:592  [s3,r1/72:/M{in-ax}] applied Raft snapshot in 0.023s
I161111 17:23:26.417802 81 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 48 [next = 589, match = 588, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 588]
I161111 17:23:26.683705 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:72 > next_replica_id:73
I161111 17:23:26.685690 4110 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:72}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:26.692927 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1f7fb2cc at index 595 in 4.823205ms.
I161111 17:23:26.698066 452 storage/store.go:2988  [s3,r1/72:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:26.700061 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 750, log entries: 49
I161111 17:23:26.786408 340 storage/store.go:2041  [s3,r1/72:/M{in-ax}] DEL 1
I161111 17:23:26.786956 4153 storage/store.go:1912  [s3] ADD 1
I161111 17:23:26.787032 4153 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:26.789479 4153 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:26.789499 4153 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:26.789522 4153 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:26.789534 4153 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:26.792901 4153 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 595, term: 6]
I161111 17:23:26.792945 4153 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 595, term: 6]
I161111 17:23:26.793016 4153 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 596, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.793050 4153 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 596, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.793100 4153 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 595] restored snapshot [index: 595, term: 6]
I161111 17:23:26.793133 4153 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 595 (id=1f7fb2cc, encoded size=16, 1 rocksdb batches, 49 log entries)
I161111 17:23:26.876729 4153 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.084s
I161111 17:23:26.876901 4153 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:26.878223 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:73
I161111 17:23:26.879763 4065 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:73}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:73}]
I161111 17:23:26.958628 273 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 49 became follower at term 6
I161111 17:23:26.958677 273 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 49 [peers: [1,2], term: 6, commit: 595, applied: 595, lastindex: 595, lastterm: 6]
I161111 17:23:26.958722 273 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 49 [logterm: 0, index: 600] rejected msgApp [logterm: 6, index: 600] from 1
I161111 17:23:26.962671 73 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 595) from 49 for index 600
I161111 17:23:26.962718 73 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 49 to [next = 596, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:26.999298 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:73 > next_replica_id:74
I161111 17:23:27.001832 4160 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:73}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:27.008885 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot b1a2b40e at index 605 in 49.11µs.
I161111 17:23:27.009401 452 storage/store.go:2988  [s3,r1/73:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.010358 452 storage/store.go:2988  [s3,r1/73:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.010845 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 762, log entries: 59
I161111 17:23:27.103858 340 storage/store.go:2041  [s3,r1/73:/M{in-ax}] DEL 1
I161111 17:23:27.105759 4228 storage/store.go:1912  [s3] ADD 1
I161111 17:23:27.105951 4228 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:27.177787 4228 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:27.177852 4228 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:27.177891 4228 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:27.177935 4228 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:27.211219 4228 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 605, term: 6]
I161111 17:23:27.211381 4228 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 605, term: 6]
I161111 17:23:27.211534 4228 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 606, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.211616 4228 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 606, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.211785 4228 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 605] restored snapshot [index: 605, term: 6]
I161111 17:23:27.211874 4228 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 605 (id=b1a2b40e, encoded size=16, 1 rocksdb batches, 59 log entries)
I161111 17:23:27.279882 4228 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.068s
I161111 17:23:27.281509 4228 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:27.281755 286 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 49 became follower at term 6
I161111 17:23:27.281797 286 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 49 [peers: [1,2], term: 6, commit: 605, applied: 605, lastindex: 605, lastterm: 6]
I161111 17:23:27.282772 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:74
I161111 17:23:27.284327 4176 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:74}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:74}]
I161111 17:23:27.288279 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.316186 245 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4a became follower at term 6
I161111 17:23:27.316235 245 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4a [peers: [1,2], term: 6, commit: 605, applied: 605, lastindex: 605, lastterm: 6]
I161111 17:23:27.316258 245 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4a [logterm: 0, index: 611] rejected msgApp [logterm: 6, index: 611] from 1
I161111 17:23:27.317192 76 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 605) from 4a for index 611
I161111 17:23:27.317223 76 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4a to [next = 606, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.318047 243 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4a [logterm: 0, index: 611] rejected msgApp [logterm: 6, index: 611] from 1
I161111 17:23:27.319808 83 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 605) from 4a for index 611
I161111 17:23:27.324654 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:74 > next_replica_id:75
I161111 17:23:27.326333 4072 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:74}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:27.328294 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 00fafbea at index 615 in 56.344µs.
I161111 17:23:27.338993 452 storage/store.go:2988  [s3,r1/74:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.342169 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 774, log entries: 69
I161111 17:23:27.365069 452 storage/store.go:2988  [s3,r1/74:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.366696 4307 raft/raft.go:568  [s3,r1/74:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:27.366775 4307 raft/raft.go:316  [s3,r1/74:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 615, applied: 615, lastindex: 615, lastterm: 6]
I161111 17:23:27.366857 4307 raft/raft.go:1106  [s3,r1/74:/M{in-ax}] ffffffffffffffff [commit: 615] ignored snapshot [index: 615, term: 6]
I161111 17:23:27.374960 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:75
I161111 17:23:27.377409 4325 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:75}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:75}]
I161111 17:23:27.396676 340 storage/store.go:2041  [s3,r1/74:/M{in-ax}] DEL 1
I161111 17:23:27.456574 250 raft/raft.go:568  [s3,r1/75:{-}] 4b became follower at term 0
I161111 17:23:27.459114 250 raft/raft.go:316  [s3,r1/75:{-}] newRaft 4b [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:27.459133 250 raft/raft.go:568  [s3,r1/75:{-}] 4b became follower at term 1
I161111 17:23:27.459155 250 raft/raft.go:710  [s3,r1/75:{-}] 4b [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:27.459168 250 raft/raft.go:568  [s3,r1/75:{-}] 4b became follower at term 6
I161111 17:23:27.484513 252 raft/raft.go:1088  [s3,r1/75:{-}] 4b [logterm: 0, index: 617] rejected msgApp [logterm: 6, index: 617] from 1
I161111 17:23:27.484691 112 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 4b for index 617
I161111 17:23:27.484715 112 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4b to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.484840 112 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 6ea599c2 at index 617 in 75.826µs.
I161111 17:23:27.484875 112 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 547, commit: 617] sent snapshot[index: 617, term: 6] to 4b [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.484892 112 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 4b [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 617]
I161111 17:23:27.519212 4356 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 777, log entries: 71
I161111 17:23:27.519816 4311 storage/store.go:1912  [s3] ADD 1
I161111 17:23:27.522481 4311 raft/raft.go:1125  [s3,r1/75:{-}] 4b [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 617, term: 6]
I161111 17:23:27.522507 4311 raft/log.go:288  [s3,r1/75:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 617, term: 6]
I161111 17:23:27.522530 4311 raft/raft.go:1135  [s3,r1/75:{-}] 4b restored progress of 1 [next = 618, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.522546 4311 raft/raft.go:1135  [s3,r1/75:{-}] 4b restored progress of 2 [next = 618, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.522562 4311 raft/raft.go:1135  [s3,r1/75:{-}] 4b restored progress of 4b [next = 618, match = 617, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.522576 4311 raft/raft.go:1102  [s3,r1/75:{-}] 4b [commit: 617] restored snapshot [index: 617, term: 6]
I161111 17:23:27.522605 4311 storage/replica_raftstorage.go:589  [s3,r1/75:{-}] applying Raft snapshot at index 617 (id=6ea599c2, encoded size=16, 1 rocksdb batches, 71 log entries)
I161111 17:23:27.559260 4311 storage/replica_raftstorage.go:592  [s3,r1/75:/M{in-ax}] applied Raft snapshot in 0.037s
I161111 17:23:27.559638 74 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 4b [next = 618, match = 617, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 617]
I161111 17:23:27.686254 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:75 > next_replica_id:76
I161111 17:23:27.688153 4369 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:75}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:27.690623 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7c1a819e at index 625 in 72.489µs.
I161111 17:23:27.692002 452 storage/store.go:2988  [s3,r1/75:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.698231 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 786, log entries: 79
I161111 17:23:27.707191 340 storage/store.go:2041  [s3,r1/75:/M{in-ax}] DEL 1
W161111 17:23:27.707494 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 75}: raft group deleted
I161111 17:23:27.712775 4335 storage/store.go:1912  [s3] ADD 1
I161111 17:23:27.712844 4335 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:27.715414 4335 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:27.715434 4335 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:27.715457 4335 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:27.715468 4335 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:27.718926 4335 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 625, term: 6]
I161111 17:23:27.719000 4335 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 625, term: 6]
I161111 17:23:27.719064 4335 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 626, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.719100 4335 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 626, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.719151 4335 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 625] restored snapshot [index: 625, term: 6]
I161111 17:23:27.719187 4335 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 625 (id=7c1a819e, encoded size=16, 1 rocksdb batches, 79 log entries)
I161111 17:23:27.757861 4335 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.038s
I161111 17:23:27.757906 4335 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:27.759834 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:76
I161111 17:23:27.761480 4305 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:76}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:76}]
I161111 17:23:27.821799 279 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4c became follower at term 6
I161111 17:23:27.821848 279 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4c [peers: [1,2], term: 6, commit: 625, applied: 625, lastindex: 625, lastterm: 6]
I161111 17:23:27.821895 279 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4c [logterm: 0, index: 627] rejected msgApp [logterm: 6, index: 627] from 1
I161111 17:23:27.822069 71 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 625) from 4c for index 627
I161111 17:23:27.822096 71 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4c to [next = 626, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:27.839882 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:76 > next_replica_id:77
I161111 17:23:27.859282 4385 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:76}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:27.874577 452 storage/store.go:2988  [s3,r1/76:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.886432 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a4a631c0 at index 632 in 58.267µs.
I161111 17:23:27.930452 452 storage/store.go:2988  [s3,r1/76:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:27.931971 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 795, log entries: 86
I161111 17:23:27.935583 340 storage/store.go:2041  [s3,r1/76:/M{in-ax}] DEL 1
I161111 17:23:27.936945 4337 storage/store.go:1912  [s3] ADD 1
I161111 17:23:27.937414 4337 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:28.056638 4337 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:28.056698 4337 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:28.056736 4337 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:28.056782 4337 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:28.079885 4337 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 632, term: 6]
I161111 17:23:28.080172 4337 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 632, term: 6]
I161111 17:23:28.080417 4337 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 633, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.080581 4337 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 633, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.098439 4337 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 632] restored snapshot [index: 632, term: 6]
I161111 17:23:28.098536 4337 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 632 (id=a4a631c0, encoded size=16, 1 rocksdb batches, 86 log entries)
I161111 17:23:28.312367 4337 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.214s
I161111 17:23:28.312412 4337 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:28.313164 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:77
I161111 17:23:28.314681 4256 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:77}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:77}]
I161111 17:23:28.358438 249 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4d became follower at term 6
I161111 17:23:28.358482 249 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4d [peers: [1,2], term: 6, commit: 632, applied: 632, lastindex: 632, lastterm: 6]
I161111 17:23:28.358526 249 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4d [logterm: 0, index: 640] rejected msgApp [logterm: 6, index: 640] from 1
I161111 17:23:28.358701 99 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 632) from 4d for index 640
I161111 17:23:28.358724 99 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4d to [next = 633, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.363712 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:77 > next_replica_id:78
I161111 17:23:28.390507 4423 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:77}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:28.399270 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 785a5e87 at index 645 in 52.374µs.
I161111 17:23:28.405358 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 810, log entries: 99
I161111 17:23:28.418298 452 storage/store.go:2988  [s3,r1/77:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:28.426500 340 storage/store.go:2041  [s3,r1/77:/M{in-ax}] DEL 1
I161111 17:23:28.427009 4477 storage/store.go:1912  [s3] ADD 1
I161111 17:23:28.427143 4477 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:28.430479 4477 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:28.430532 4477 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:28.430570 4477 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:28.430597 4477 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:28.432929 4477 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 645, term: 6]
I161111 17:23:28.432968 4477 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 645, term: 6]
I161111 17:23:28.433007 4477 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 646, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.433049 4477 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 646, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.433077 4477 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 645] restored snapshot [index: 645, term: 6]
I161111 17:23:28.433107 4477 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 645 (id=785a5e87, encoded size=16, 1 rocksdb batches, 99 log entries)
I161111 17:23:28.507632 4477 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.074s
I161111 17:23:28.510559 4477 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:28.511706 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:78
I161111 17:23:28.513339 4410 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:78}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:78}]
I161111 17:23:28.538419 264 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4e became follower at term 6
I161111 17:23:28.538459 264 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4e [peers: [1,2], term: 6, commit: 645, applied: 645, lastindex: 645, lastterm: 6]
I161111 17:23:28.538502 264 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4e [logterm: 0, index: 647] rejected msgApp [logterm: 6, index: 647] from 1
I161111 17:23:28.538671 77 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 645) from 4e for index 647
I161111 17:23:28.538694 77 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4e to [next = 646, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.564908 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:78 > next_replica_id:79
I161111 17:23:28.566916 4517 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:78}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:28.567390 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 547-651
I161111 17:23:28.569640 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot c2e2973f at index 652 in 55.598µs.
I161111 17:23:28.602017 452 storage/store.go:2988  [s3,r1/78:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:28.602194 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 819, log entries: 106
I161111 17:23:28.648585 340 storage/store.go:2041  [s3,r1/78:/M{in-ax}] DEL 1
I161111 17:23:28.659537 4538 storage/store.go:1912  [s3] ADD 1
I161111 17:23:28.659639 4538 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:28.747141 4538 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:28.747185 4538 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:28.747221 4538 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:28.747246 4538 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:28.790990 4538 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 652, term: 6]
I161111 17:23:28.791083 4538 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 652, term: 6]
I161111 17:23:28.791319 4538 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 653, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.791356 4538 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 653, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:28.791422 4538 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 652] restored snapshot [index: 652, term: 6]
I161111 17:23:28.791606 4538 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 652 (id=c2e2973f, encoded size=16, 1 rocksdb batches, 106 log entries)
I161111 17:23:29.155340 4538 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.364s
I161111 17:23:29.155388 4538 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
W161111 17:23:29.155521 4538 storage/replica.go:554  [s3,r1/?:/M{in-ax}] raftMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest for 506.07214ms (>500ms):
goroutine 4538 [running]:
runtime/debug.Stack(0x1c197a0, 0x1c197e0, 0x48)
	/home/tamird/goroot/src/runtime/debug/stack.go:24 +0x79
github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x1e2a0c4c)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:65 +0xe1
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc420361878)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:92 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420094000, 0x7f530f21b878, 0xc4203ccd50, 0xc4203d98b8, 0x1e425a053f97e2c2, 0x609b9e5a2c6bb29b, 0x1, 0x277bef8, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2925 +0x83c
github.com/cockroachdb/cockroach/pkg/storage.(*Store).HandleSnapshot(0xc420094000, 0xc4203d9860, 0x7f530d5c1028, 0xc420328e20, 0x0, 0x0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2610 +0x746
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftSnapshot.func1.1(0x23c0440, 0xc420328e20, 0xc4205f70e0, 0xc420644738, 0xc420644740)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:361 +0x17f
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftSnapshot.func1(0x7f530f21b878, 0xc42089ecc0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:362 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4200c6000, 0x20b8720, 0x19, 0x16b, 0x0, 0x0, 0xc42089ee40, 0x7f530f21b878, 0xc42089ecc0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234
I161111 17:23:29.156226 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:79
I161111 17:23:29.157733 4528 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:79}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:79}]
I161111 17:23:29.253979 282 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 4f became follower at term 6
I161111 17:23:29.254034 282 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 4f [peers: [1,2], term: 6, commit: 652, applied: 652, lastindex: 652, lastterm: 6]
I161111 17:23:29.258890 284 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 4f [logterm: 0, index: 664] rejected msgApp [logterm: 6, index: 664] from 1
I161111 17:23:29.259080 94 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 652) from 4f for index 664
I161111 17:23:29.259107 94 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 4f to [next = 653, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.330074 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:79 > next_replica_id:80
I161111 17:23:29.332106 4580 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:79}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:29.334727 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot db30e2af at index 669 in 50.971µs.
I161111 17:23:29.337231 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 837, log entries: 19
I161111 17:23:29.340607 452 storage/store.go:2988  [s3,r1/79:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:29.341494 4346 raft/raft.go:568  [s3,r1/79:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:29.341568 4346 raft/raft.go:316  [s3,r1/79:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 669, applied: 669, lastindex: 669, lastterm: 6]
I161111 17:23:29.341729 4346 raft/raft.go:1106  [s3,r1/79:/M{in-ax}] ffffffffffffffff [commit: 669] ignored snapshot [index: 669, term: 6]
I161111 17:23:29.345119 340 storage/store.go:2041  [s3,r1/79:/M{in-ax}] DEL 1
I161111 17:23:29.349309 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:80
I161111 17:23:29.351115 4432 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:80}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:80}]
W161111 17:23:29.354538 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 79}: raft group deleted
I161111 17:23:29.441679 255 raft/raft.go:568  [s3,r1/80:{-}] 50 became follower at term 0
I161111 17:23:29.442184 255 raft/raft.go:316  [s3,r1/80:{-}] newRaft 50 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:29.442199 255 raft/raft.go:568  [s3,r1/80:{-}] 50 became follower at term 1
I161111 17:23:29.442221 255 raft/raft.go:710  [s3,r1/80:{-}] 50 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:29.442238 255 raft/raft.go:568  [s3,r1/80:{-}] 50 became follower at term 6
I161111 17:23:29.442256 255 raft/raft.go:1088  [s3,r1/80:{-}] 50 [logterm: 0, index: 671] rejected msgApp [logterm: 6, index: 671] from 1
I161111 17:23:29.442481 82 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 50 for index 671
I161111 17:23:29.442507 82 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 50 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.442620 82 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 88c72e44 at index 671 in 65.905µs.
I161111 17:23:29.442655 82 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 651, commit: 671] sent snapshot[index: 671, term: 6] to 50 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.442672 82 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 50 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 671]
I161111 17:23:29.456595 4658 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 840, log entries: 21
I161111 17:23:29.460384 4664 storage/store.go:1912  [s3] ADD 1
I161111 17:23:29.460964 4664 raft/raft.go:1125  [s3,r1/80:{-}] 50 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 671, term: 6]
I161111 17:23:29.461003 4664 raft/log.go:288  [s3,r1/80:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 671, term: 6]
I161111 17:23:29.461036 4664 raft/raft.go:1135  [s3,r1/80:{-}] 50 restored progress of 1 [next = 672, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.461066 4664 raft/raft.go:1135  [s3,r1/80:{-}] 50 restored progress of 2 [next = 672, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.461092 4664 raft/raft.go:1135  [s3,r1/80:{-}] 50 restored progress of 50 [next = 672, match = 671, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.461113 4664 raft/raft.go:1102  [s3,r1/80:{-}] 50 [commit: 671] restored snapshot [index: 671, term: 6]
I161111 17:23:29.461174 4664 storage/replica_raftstorage.go:589  [s3,r1/80:{-}] applying Raft snapshot at index 671 (id=88c72e44, encoded size=16, 1 rocksdb batches, 21 log entries)
I161111 17:23:29.488517 4664 storage/replica_raftstorage.go:592  [s3,r1/80:/M{in-ax}] applied Raft snapshot in 0.027s
I161111 17:23:29.489203 4658 raft/raft.go:933  [s1,r1/1:/M{in-ax}] 1 snapshot succeeded, resumed sending replication messages to 50 [next = 672, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.492870 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:80 > next_replica_id:81
I161111 17:23:29.496164 4676 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:80}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:29.499772 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 77b3f954 at index 676 in 54µs.
I161111 17:23:29.505950 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 846, log entries: 26
I161111 17:23:29.506156 452 storage/store.go:2988  [s3,r1/80:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:29.518855 340 storage/store.go:2041  [s3,r1/80:/M{in-ax}] DEL 1
I161111 17:23:29.519187 3648 storage/store.go:1912  [s3] ADD 1
I161111 17:23:29.519420 3648 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:29.519974 3648 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:29.520003 3648 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:29.520041 3648 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:29.520062 3648 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:29.520553 3648 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 676, term: 6]
I161111 17:23:29.520592 3648 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 676, term: 6]
I161111 17:23:29.520648 3648 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 677, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.520678 3648 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 677, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.520702 3648 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 676] restored snapshot [index: 676, term: 6]
I161111 17:23:29.520730 3648 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 676 (id=77b3f954, encoded size=16, 1 rocksdb batches, 26 log entries)
I161111 17:23:29.524939 3648 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.004s
I161111 17:23:29.551776 3648 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:29.552898 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:81
I161111 17:23:29.554497 4692 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:81}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:81}]
I161111 17:23:29.603725 268 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 51 became follower at term 6
I161111 17:23:29.603769 268 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 51 [peers: [1,2], term: 6, commit: 676, applied: 676, lastindex: 676, lastterm: 6]
I161111 17:23:29.618517 270 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 51 [logterm: 0, index: 678] rejected msgApp [logterm: 6, index: 678] from 1
I161111 17:23:29.618686 66 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 676) from 51 for index 678
I161111 17:23:29.618711 66 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 51 to [next = 677, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.630879 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:81 > next_replica_id:82
I161111 17:23:29.632799 4685 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:81}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:29.635363 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1d597827 at index 681 in 49.075µs.
I161111 17:23:29.636733 452 storage/store.go:2988  [s3,r1/81:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:29.642251 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 853, log entries: 31
I161111 17:23:29.649999 452 storage/store.go:2988  [s3,r1/81:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:29.662901 340 storage/store.go:2041  [s3,r1/81:/M{in-ax}] DEL 1
I161111 17:23:29.663485 4649 storage/store.go:1912  [s3] ADD 1
I161111 17:23:29.663572 4649 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:29.664287 4649 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:29.664305 4649 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:29.664326 4649 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:29.664338 4649 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:29.664841 4649 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 681, term: 6]
I161111 17:23:29.664861 4649 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 681, term: 6]
I161111 17:23:29.664881 4649 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 682, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.664896 4649 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 682, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.664908 4649 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 681] restored snapshot [index: 681, term: 6]
I161111 17:23:29.664925 4649 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 681 (id=1d597827, encoded size=16, 1 rocksdb batches, 31 log entries)
W161111 17:23:29.668528 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 81}: raft group deleted
I161111 17:23:29.670218 4649 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.005s
I161111 17:23:29.670270 4649 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:29.670912 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:82
I161111 17:23:29.672455 4743 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:82}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:82}]
I161111 17:23:29.754804 282 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 52 became follower at term 6
I161111 17:23:29.754853 282 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 52 [peers: [1,2], term: 6, commit: 681, applied: 681, lastindex: 681, lastterm: 6]
I161111 17:23:29.789302 281 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 52 [logterm: 0, index: 685] rejected msgApp [logterm: 6, index: 685] from 1
I161111 17:23:29.789477 74 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 681) from 52 for index 685
I161111 17:23:29.789509 74 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 52 to [next = 682, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.858940 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:82 > next_replica_id:83
I161111 17:23:29.863576 4656 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:82}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:29.883223 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ab932791 at index 690 in 56.377µs.
I161111 17:23:29.891798 340 storage/store.go:2041  [s3,r1/82:/M{in-ax}] DEL 1
I161111 17:23:29.896407 4559 storage/store.go:1912  [s3] ADD 1
I161111 17:23:29.896501 4559 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:29.897769 4559 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:29.897790 4559 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:29.897818 4559 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:29.897833 4559 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:29.898940 4559 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 690, term: 6]
I161111 17:23:29.898964 4559 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 690, term: 6]
I161111 17:23:29.898986 4559 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 691, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.899001 4559 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 691, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:29.899013 4559 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 690] restored snapshot [index: 690, term: 6]
I161111 17:23:29.899039 4559 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 690 (id=ab932791, encoded size=16, 1 rocksdb batches, 40 log entries)
E161111 17:23:29.901843 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:29.901885 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:29.901915 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:29.901946 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
E161111 17:23:29.902301 452 storage/store.go:2986  [s3,r1/?:{-}] unable to add to replica GC queue: replica not initialized
I161111 17:23:29.908895 4559 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.010s
I161111 17:23:29.908936 4559 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:29.909182 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 864, log entries: 40
I161111 17:23:29.909702 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:83
I161111 17:23:29.911261 4786 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:83}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:83}]
I161111 17:23:29.978605 250 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 53 became follower at term 6
I161111 17:23:29.978650 250 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 53 [peers: [1,2], term: 6, commit: 690, applied: 690, lastindex: 690, lastterm: 6]
I161111 17:23:29.978698 250 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 53 [logterm: 0, index: 692] rejected msgApp [logterm: 6, index: 692] from 1
I161111 17:23:29.978884 67 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 690) from 53 for index 692
I161111 17:23:29.978908 67 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 53 to [next = 691, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.011000 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:83 > next_replica_id:84
I161111 17:23:30.012949 4795 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:83}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:30.018212 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 06a41b1a at index 697 in 54.481µs.
I161111 17:23:30.031540 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 873, log entries: 47
I161111 17:23:30.031822 452 storage/store.go:2988  [s3,r1/83:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:30.041155 340 storage/store.go:2041  [s3,r1/83:/M{in-ax}] DEL 1
I161111 17:23:30.041736 4755 storage/store.go:1912  [s3] ADD 1
I161111 17:23:30.041824 4755 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:30.043230 4755 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:30.043249 4755 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:30.043274 4755 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:30.043286 4755 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:30.044451 4755 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 697, term: 6]
I161111 17:23:30.044474 4755 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 697, term: 6]
I161111 17:23:30.044495 4755 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 698, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.044512 4755 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 698, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.044524 4755 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 697] restored snapshot [index: 697, term: 6]
I161111 17:23:30.044543 4755 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 697 (id=06a41b1a, encoded size=16, 1 rocksdb batches, 47 log entries)
I161111 17:23:30.054857 4755 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.010s
I161111 17:23:30.056012 4755 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:30.056316 271 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 53 became follower at term 6
I161111 17:23:30.056377 271 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 53 [peers: [1,2], term: 6, commit: 697, applied: 697, lastindex: 697, lastterm: 6]
I161111 17:23:30.057019 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:84
I161111 17:23:30.059197 4837 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:84}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:84}]
I161111 17:23:30.087357 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:30.137192 340 storage/store.go:2041  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:30.155310 275 raft/raft.go:568  [s3,r1/84:{-}] 54 became follower at term 0
I161111 17:23:30.156765 275 raft/raft.go:316  [s3,r1/84:{-}] newRaft 54 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:30.156782 275 raft/raft.go:568  [s3,r1/84:{-}] 54 became follower at term 1
I161111 17:23:30.156804 275 raft/raft.go:710  [s3,r1/84:{-}] 54 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:30.156826 275 raft/raft.go:568  [s3,r1/84:{-}] 54 became follower at term 6
I161111 17:23:30.158861 269 raft/raft.go:1088  [s3,r1/84:{-}] 54 [logterm: 0, index: 699] rejected msgApp [logterm: 6, index: 699] from 1
I161111 17:23:30.159013 109 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 54 for index 699
I161111 17:23:30.159046 109 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 54 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.159153 109 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot c138d4c3 at index 699 in 62.869µs.
I161111 17:23:30.159187 109 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 651, commit: 699] sent snapshot[index: 699, term: 6] to 54 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.159204 109 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 54 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 699]
I161111 17:23:30.210083 4797 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 876, log entries: 49
I161111 17:23:30.210270 4815 storage/store.go:1912  [s3] ADD 1
I161111 17:23:30.211944 4815 raft/raft.go:1125  [s3,r1/84:{-}] 54 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 699, term: 6]
I161111 17:23:30.211996 4815 raft/log.go:288  [s3,r1/84:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 699, term: 6]
I161111 17:23:30.212037 4815 raft/raft.go:1135  [s3,r1/84:{-}] 54 restored progress of 1 [next = 700, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.212076 4815 raft/raft.go:1135  [s3,r1/84:{-}] 54 restored progress of 2 [next = 700, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.212102 4815 raft/raft.go:1135  [s3,r1/84:{-}] 54 restored progress of 54 [next = 700, match = 699, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.212143 4815 raft/raft.go:1102  [s3,r1/84:{-}] 54 [commit: 699] restored snapshot [index: 699, term: 6]
I161111 17:23:30.212183 4815 storage/replica_raftstorage.go:589  [s3,r1/84:{-}] applying Raft snapshot at index 699 (id=c138d4c3, encoded size=16, 1 rocksdb batches, 49 log entries)
I161111 17:23:30.255002 4815 storage/replica_raftstorage.go:592  [s3,r1/84:/M{in-ax}] applied Raft snapshot in 0.043s
I161111 17:23:30.255778 85 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 54 [next = 700, match = 699, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 699]
I161111 17:23:30.383250 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:84 > next_replica_id:85
I161111 17:23:30.385091 4885 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:84}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:30.389731 452 storage/store.go:2988  [s3,r1/84:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:30.390360 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot d1d8c7aa at index 707 in 51.923µs.
I161111 17:23:30.394958 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 885, log entries: 57
I161111 17:23:30.400396 340 storage/store.go:2041  [s3,r1/84:/M{in-ax}] DEL 1
I161111 17:23:30.411741 4915 storage/store.go:1912  [s3] ADD 1
I161111 17:23:30.411825 4915 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:30.413397 4915 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:30.413415 4915 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:30.413437 4915 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:30.413448 4915 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:30.414637 4915 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 707, term: 6]
I161111 17:23:30.414657 4915 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 707, term: 6]
I161111 17:23:30.414676 4915 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 708, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.414690 4915 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 708, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.414703 4915 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 707] restored snapshot [index: 707, term: 6]
I161111 17:23:30.414719 4915 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 707 (id=d1d8c7aa, encoded size=16, 1 rocksdb batches, 57 log entries)
I161111 17:23:30.434364 4915 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.020s
I161111 17:23:30.437621 4915 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:30.438609 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:85
I161111 17:23:30.440005 4889 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:85}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:85}]
I161111 17:23:30.520681 259 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 55 became follower at term 6
I161111 17:23:30.520724 259 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 55 [peers: [1,2], term: 6, commit: 707, applied: 707, lastindex: 707, lastterm: 6]
I161111 17:23:30.520770 259 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 55 [logterm: 0, index: 709] rejected msgApp [logterm: 6, index: 709] from 1
I161111 17:23:30.520957 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 707) from 55 for index 709
I161111 17:23:30.520992 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 55 to [next = 708, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.525135 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:85 > next_replica_id:86
I161111 17:23:30.529640 4859 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:85}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:30.531580 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 96cce509 at index 712 in 54.212µs.
I161111 17:23:30.639233 452 storage/store.go:2988  [s3,r1/85:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:30.639390 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 892, log entries: 62
I161111 17:23:30.644969 340 storage/store.go:2041  [s3,r1/85:/M{in-ax}] DEL 1
I161111 17:23:30.645514 4870 storage/store.go:1912  [s3] ADD 1
I161111 17:23:30.645602 4870 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:30.647201 4870 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:30.647220 4870 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:30.647243 4870 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:30.647255 4870 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:30.648739 4870 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 712, term: 6]
I161111 17:23:30.648762 4870 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 712, term: 6]
I161111 17:23:30.648786 4870 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 713, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.648802 4870 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 713, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.648817 4870 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 712] restored snapshot [index: 712, term: 6]
I161111 17:23:30.648835 4870 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 712 (id=96cce509, encoded size=16, 1 rocksdb batches, 62 log entries)
W161111 17:23:30.659388 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 85}: raft group deleted
I161111 17:23:30.674130 4870 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.025s
I161111 17:23:30.683447 4870 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:30.684417 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:86
I161111 17:23:30.685945 4877 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:86}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:86}]
I161111 17:23:30.698152 272 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 56 became follower at term 6
I161111 17:23:30.698196 272 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 56 [peers: [1,2], term: 6, commit: 712, applied: 712, lastindex: 712, lastterm: 6]
I161111 17:23:30.698261 272 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 56 [logterm: 0, index: 716] rejected msgApp [logterm: 6, index: 716] from 1
I161111 17:23:30.698467 105 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 712) from 56 for index 716
I161111 17:23:30.698494 105 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 56 to [next = 713, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.709694 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:86 > next_replica_id:87
I161111 17:23:30.711544 4984 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:86}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:30.714706 452 storage/store.go:2988  [s3,r1/86:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:30.773409 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot cf384f42 at index 721 in 62.207µs.
I161111 17:23:30.788419 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 903, log entries: 71
I161111 17:23:30.861743 340 storage/store.go:2041  [s3,r1/86:/M{in-ax}] DEL 1
W161111 17:23:30.862068 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 86}: raft group deleted
I161111 17:23:30.880647 4934 storage/store.go:1912  [s3] ADD 1
I161111 17:23:30.880730 4934 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:30.882372 4934 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:30.882397 4934 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:30.882424 4934 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:30.882439 4934 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:30.883768 4934 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 721, term: 6]
I161111 17:23:30.883789 4934 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 721, term: 6]
I161111 17:23:30.883809 4934 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 722, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.883824 4934 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 722, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:30.883835 4934 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 721] restored snapshot [index: 721, term: 6]
I161111 17:23:30.883853 4934 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 721 (id=cf384f42, encoded size=16, 1 rocksdb batches, 71 log entries)
I161111 17:23:31.113154 4934 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.229s
I161111 17:23:31.113196 4934 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:31.113989 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:87
I161111 17:23:31.115594 4907 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:87}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:87}]
I161111 17:23:31.240946 287 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 57 became follower at term 6
I161111 17:23:31.241076 287 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 57 [peers: [1,2], term: 6, commit: 721, applied: 721, lastindex: 721, lastterm: 6]
I161111 17:23:31.241097 287 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 57 [logterm: 0, index: 729] rejected msgApp [logterm: 6, index: 729] from 1
I161111 17:23:31.241253 84 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 721) from 57 for index 729
I161111 17:23:31.241282 84 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 57 to [next = 722, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.277409 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:87 > next_replica_id:88
I161111 17:23:31.282094 5052 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:87}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:31.284882 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 52d4e2ce at index 734 in 54.216µs.
I161111 17:23:31.287135 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 918, log entries: 84
I161111 17:23:31.287351 5085 raft/raft.go:568  [s3,r1/87:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:31.287387 5085 raft/raft.go:316  [s3,r1/87:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,57], term: 6, commit: 733, applied: 733, lastindex: 733, lastterm: 6]
I161111 17:23:31.287443 5085 raft/raft.go:1125  [s3,r1/87:/M{in-ax}] ffffffffffffffff [commit: 733, lastindex: 733, lastterm: 6] starts to restore snapshot [index: 734, term: 6]
I161111 17:23:31.287460 5085 raft/log.go:288  [s3,r1/87:/M{in-ax}] log [committed=733, applied=733, unstable.offset=734, len(unstable.Entries)=0] starts to restore snapshot [index: 734, term: 6]
I161111 17:23:31.287479 5085 raft/raft.go:1135  [s3,r1/87:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 735, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.287495 5085 raft/raft.go:1135  [s3,r1/87:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 735, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.287509 5085 raft/raft.go:1102  [s3,r1/87:/M{in-ax}] ffffffffffffffff [commit: 734] restored snapshot [index: 734, term: 6]
I161111 17:23:31.287527 5085 storage/replica_raftstorage.go:589  [s3,r1/87:/M{in-ax}] applying Raft snapshot at index 734 (id=52d4e2ce, encoded size=16, 1 rocksdb batches, 84 log entries)
I161111 17:23:31.289003 452 storage/store.go:2988  [s3,r1/87:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:31.539902 5085 storage/replica_raftstorage.go:592  [s3,r1/87:/M{in-ax}] applied Raft snapshot in 0.252s
I161111 17:23:31.540231 246 raft/raft.go:568  [s3,r1/87:/M{in-ax}] 57 became follower at term 6
I161111 17:23:31.540376 246 raft/raft.go:316  [s3,r1/87:/M{in-ax}] newRaft 57 [peers: [1,2], term: 6, commit: 734, applied: 734, lastindex: 734, lastterm: 6]
I161111 17:23:31.544523 452 storage/store.go:2988  [s3,r1/87:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:31.556589 340 storage/store.go:2041  [s3,r1/87:/M{in-ax}] DEL 1
I161111 17:23:31.557698 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:88
I161111 17:23:31.559424 5057 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:88}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:88}]
I161111 17:23:31.621725 252 raft/raft.go:568  [s3,r1/88:{-}] 58 became follower at term 0
I161111 17:23:31.682042 252 raft/raft.go:316  [s3,r1/88:{-}] newRaft 58 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:31.682076 252 raft/raft.go:568  [s3,r1/88:{-}] 58 became follower at term 1
I161111 17:23:31.682107 252 raft/raft.go:710  [s3,r1/88:{-}] 58 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:31.682125 252 raft/raft.go:568  [s3,r1/88:{-}] 58 became follower at term 6
I161111 17:23:31.682149 252 raft/raft.go:1088  [s3,r1/88:{-}] 58 [logterm: 0, index: 739] rejected msgApp [logterm: 6, index: 739] from 1
I161111 17:23:31.682217 252 raft/raft.go:1088  [s3,r1/88:{-}] 58 [logterm: 0, index: 739] rejected msgApp [logterm: 6, index: 739] from 1
I161111 17:23:31.682590 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 58 for index 739
I161111 17:23:31.682640 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 58 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.682834 100 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1cf78912 at index 742 in 100.743µs.
I161111 17:23:31.682892 100 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 651, commit: 742] sent snapshot[index: 742, term: 6] to 58 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.682917 100 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 58 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 742]
I161111 17:23:31.682963 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 58 for index 739
I161111 17:23:31.756207 5086 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 927, log entries: 92
I161111 17:23:31.756313 5089 storage/store.go:1912  [s3] ADD 1
I161111 17:23:31.758069 5089 raft/raft.go:1125  [s3,r1/88:{-}] 58 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 742, term: 6]
I161111 17:23:31.758092 5089 raft/log.go:288  [s3,r1/88:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 742, term: 6]
I161111 17:23:31.758112 5089 raft/raft.go:1135  [s3,r1/88:{-}] 58 restored progress of 1 [next = 743, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.758127 5089 raft/raft.go:1135  [s3,r1/88:{-}] 58 restored progress of 2 [next = 743, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.758140 5089 raft/raft.go:1135  [s3,r1/88:{-}] 58 restored progress of 58 [next = 743, match = 742, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:31.758151 5089 raft/raft.go:1102  [s3,r1/88:{-}] 58 [commit: 742] restored snapshot [index: 742, term: 6]
I161111 17:23:31.758178 5089 storage/replica_raftstorage.go:589  [s3,r1/88:{-}] applying Raft snapshot at index 742 (id=1cf78912, encoded size=16, 1 rocksdb batches, 92 log entries)
I161111 17:23:31.797662 5089 storage/replica_raftstorage.go:592  [s3,r1/88:/M{in-ax}] applied Raft snapshot in 0.039s
I161111 17:23:31.798066 79 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 58 [next = 743, match = 742, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 742]
I161111 17:23:31.834788 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:88 > next_replica_id:89
I161111 17:23:31.836712 5143 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:88}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:31.838788 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 63df7b00 at index 747 in 53.94µs.
I161111 17:23:31.842522 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 933, log entries: 97
I161111 17:23:31.846148 452 storage/store.go:2988  [s3,r1/88:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:31.847972 5115 raft/raft.go:568  [s3,r1/88:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:31.848053 5115 raft/raft.go:316  [s3,r1/88:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 747, applied: 747, lastindex: 747, lastterm: 6]
I161111 17:23:31.848143 5115 raft/raft.go:1106  [s3,r1/88:/M{in-ax}] ffffffffffffffff [commit: 747] ignored snapshot [index: 747, term: 6]
I161111 17:23:31.851863 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:89
I161111 17:23:31.854385 5117 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:89}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:89}]
I161111 17:23:31.860984 340 storage/store.go:2041  [s3,r1/88:/M{in-ax}] DEL 1
I161111 17:23:31.954044 266 raft/raft.go:568  [s3,r1/89:{-}] 59 became follower at term 0
I161111 17:23:31.955870 266 raft/raft.go:316  [s3,r1/89:{-}] newRaft 59 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:31.955887 266 raft/raft.go:568  [s3,r1/89:{-}] 59 became follower at term 1
I161111 17:23:31.955908 266 raft/raft.go:710  [s3,r1/89:{-}] 59 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:31.955920 266 raft/raft.go:568  [s3,r1/89:{-}] 59 became follower at term 6
I161111 17:23:32.004542 268 raft/raft.go:1088  [s3,r1/89:{-}] 59 [logterm: 0, index: 749] rejected msgApp [logterm: 6, index: 749] from 1
I161111 17:23:32.004767 110 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 59 for index 749
I161111 17:23:32.004799 110 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 59 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.004946 110 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 97970a31 at index 752 in 88.472µs.
I161111 17:23:32.004987 110 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 651, commit: 752] sent snapshot[index: 752, term: 6] to 59 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.005005 110 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 59 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 752]
I161111 17:23:32.018090 5189 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 939, log entries: 102
I161111 17:23:32.018350 5192 storage/store.go:1912  [s3] ADD 1
I161111 17:23:32.025717 5192 raft/raft.go:1125  [s3,r1/89:{-}] 59 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 752, term: 6]
I161111 17:23:32.025764 5192 raft/log.go:288  [s3,r1/89:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 752, term: 6]
I161111 17:23:32.025878 5192 raft/raft.go:1135  [s3,r1/89:{-}] 59 restored progress of 1 [next = 753, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.026206 5192 raft/raft.go:1135  [s3,r1/89:{-}] 59 restored progress of 2 [next = 753, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.026239 5192 raft/raft.go:1135  [s3,r1/89:{-}] 59 restored progress of 59 [next = 753, match = 752, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.026264 5192 raft/raft.go:1102  [s3,r1/89:{-}] 59 [commit: 752] restored snapshot [index: 752, term: 6]
I161111 17:23:32.026307 5192 storage/replica_raftstorage.go:589  [s3,r1/89:{-}] applying Raft snapshot at index 752 (id=97970a31, encoded size=16, 1 rocksdb batches, 102 log entries)
I161111 17:23:32.158874 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 651-755
I161111 17:23:32.265897 5192 storage/replica_raftstorage.go:592  [s3,r1/89:/M{in-ax}] applied Raft snapshot in 0.240s
I161111 17:23:32.266277 90 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 59 [next = 753, match = 752, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 752]
I161111 17:23:32.266313 90 raft/raft.go:389  [s1,r1/1:/M{in-ax}] 1 failed to send snapshot to 59 because snapshot is temporarily unavailable
I161111 17:23:32.302009 91 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a805ca57 at index 756 in 90.14µs.
I161111 17:23:32.302079 91 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 755, commit: 756] sent snapshot[index: 756, term: 6] to 59 [next = 753, match = 752, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.302099 91 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 59 [next = 753, match = 752, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 756]
I161111 17:23:32.304080 5151 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 942, log entries: 2
I161111 17:23:32.304350 5234 raft/raft.go:1125  [s3,r1/89:/M{in-ax}] 59 [commit: 752, lastindex: 752, lastterm: 6] starts to restore snapshot [index: 756, term: 6]
I161111 17:23:32.304369 5234 raft/log.go:288  [s3,r1/89:/M{in-ax}] log [committed=752, applied=752, unstable.offset=753, len(unstable.Entries)=0] starts to restore snapshot [index: 756, term: 6]
I161111 17:23:32.304387 5234 raft/raft.go:1135  [s3,r1/89:/M{in-ax}] 59 restored progress of 1 [next = 757, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.304403 5234 raft/raft.go:1135  [s3,r1/89:/M{in-ax}] 59 restored progress of 2 [next = 757, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.304418 5234 raft/raft.go:1135  [s3,r1/89:/M{in-ax}] 59 restored progress of 59 [next = 757, match = 756, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.304429 5234 raft/raft.go:1102  [s3,r1/89:/M{in-ax}] 59 [commit: 756] restored snapshot [index: 756, term: 6]
I161111 17:23:32.304454 5234 storage/replica_raftstorage.go:589  [s3,r1/89:/M{in-ax}] applying Raft snapshot at index 756 (id=a805ca57, encoded size=16, 1 rocksdb batches, 2 log entries)
I161111 17:23:32.404090 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:89 > next_replica_id:90
I161111 17:23:32.407185 5246 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:89}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:32.504062 5234 storage/replica_raftstorage.go:592  [s3,r1/89:/M{in-ax}] applied Raft snapshot in 0.200s
I161111 17:23:32.504996 452 storage/store.go:2988  [s3,r1/89:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:32.514373 5151 raft/raft.go:854  [s1,r1/1:/M{in-ax}] 1 no progress available for 59
I161111 17:23:32.514535 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 72449392 at index 762 in 81.897µs.
I161111 17:23:32.541161 452 storage/store.go:2988  [s3,r1/89:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:32.549438 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 949, log entries: 8
I161111 17:23:32.730450 340 storage/store.go:2041  [s3,r1/89:/M{in-ax}] DEL 1
I161111 17:23:32.731288 5181 storage/store.go:1912  [s3] ADD 1
I161111 17:23:32.731459 5181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:32.734162 5181 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:32.734193 5181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:32.734246 5181 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:32.734267 5181 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:32.736340 5181 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 762, term: 6]
I161111 17:23:32.736376 5181 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 762, term: 6]
I161111 17:23:32.736408 5181 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 763, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.736434 5181 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 763, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:32.736524 5181 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 762] restored snapshot [index: 762, term: 6]
I161111 17:23:32.736574 5181 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 762 (id=72449392, encoded size=16, 1 rocksdb batches, 8 log entries)
I161111 17:23:33.049817 5181 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.313s
I161111 17:23:33.049857 5181 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:33.050539 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:90
I161111 17:23:33.051978 5259 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:90}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:90}]
I161111 17:23:33.088362 279 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5a became follower at term 6
I161111 17:23:33.088514 279 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5a [peers: [1,2], term: 6, commit: 762, applied: 762, lastindex: 762, lastterm: 6]
I161111 17:23:33.088537 279 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5a [logterm: 0, index: 773] rejected msgApp [logterm: 6, index: 773] from 1
I161111 17:23:33.089220 96 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 762) from 5a for index 773
I161111 17:23:33.089254 96 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5a to [next = 763, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.111851 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:90 > next_replica_id:91
I161111 17:23:33.115175 5207 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:90}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:33.117990 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot fe53995c at index 777 in 82.453µs.
I161111 17:23:33.120256 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 966, log entries: 23
I161111 17:23:33.121211 452 storage/store.go:2988  [s3,r1/90:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:33.326279 340 storage/store.go:2041  [s3,r1/90:/M{in-ax}] DEL 1
W161111 17:23:33.328665 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 90}: raft group deleted
I161111 17:23:33.329809 2109 storage/store.go:1912  [s3] ADD 1
I161111 17:23:33.330104 2109 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:33.336929 2109 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:33.351679 2109 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:33.364563 2109 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:33.364602 2109 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:33.378767 2109 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 777, term: 6]
I161111 17:23:33.378821 2109 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 777, term: 6]
I161111 17:23:33.378853 2109 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 778, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.378879 2109 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 778, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.378900 2109 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 777] restored snapshot [index: 777, term: 6]
I161111 17:23:33.378929 2109 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 777 (id=fe53995c, encoded size=16, 1 rocksdb batches, 23 log entries)
I161111 17:23:33.497413 2109 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.118s
I161111 17:23:33.501014 2109 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:33.502252 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:91
I161111 17:23:33.513309 5215 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:91}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:91}]
I161111 17:23:33.580788 243 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5b became follower at term 6
I161111 17:23:33.580830 243 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5b [peers: [1,2], term: 6, commit: 777, applied: 777, lastindex: 777, lastterm: 6]
I161111 17:23:33.580874 243 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5b [logterm: 0, index: 785] rejected msgApp [logterm: 6, index: 785] from 1
I161111 17:23:33.581046 76 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 777) from 5b for index 785
I161111 17:23:33.581072 76 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5b to [next = 778, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.610335 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:91 > next_replica_id:92
I161111 17:23:33.612734 5364 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:91}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:33.634291 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 62f4c770 at index 789 in 58.333µs.
I161111 17:23:33.636699 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 980, log entries: 35
I161111 17:23:33.637550 452 storage/store.go:2988  [s3,r1/91:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:33.695044 340 storage/store.go:2041  [s3,r1/91:/M{in-ax}] DEL 1
I161111 17:23:33.697706 5383 storage/store.go:1912  [s3] ADD 1
I161111 17:23:33.697811 5383 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:33.699721 5383 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:33.704690 5383 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:33.704775 5383 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:33.704898 5383 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:33.706905 5383 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 789, term: 6]
I161111 17:23:33.706943 5383 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 789, term: 6]
I161111 17:23:33.706973 5383 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 790, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.706997 5383 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 790, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.707019 5383 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 789] restored snapshot [index: 789, term: 6]
I161111 17:23:33.707047 5383 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 789 (id=62f4c770, encoded size=16, 1 rocksdb batches, 35 log entries)
I161111 17:23:33.934164 5383 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.227s
I161111 17:23:33.934303 5383 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:33.934459 256 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5b became follower at term 6
I161111 17:23:33.934498 256 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5b [peers: [1,2], term: 6, commit: 789, applied: 789, lastindex: 789, lastterm: 6]
I161111 17:23:33.935735 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:92
I161111 17:23:33.937177 5325 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:92}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:92}]
I161111 17:23:33.942358 259 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5c became follower at term 6
I161111 17:23:33.942403 259 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5c [peers: [1,2], term: 6, commit: 789, applied: 789, lastindex: 789, lastterm: 6]
I161111 17:23:33.942448 259 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5c [logterm: 0, index: 795] rejected msgApp [logterm: 6, index: 795] from 1
I161111 17:23:33.942705 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:33.943069 105 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 789) from 5c for index 795
I161111 17:23:33.943094 105 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5c to [next = 790, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:33.970008 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:92 > next_replica_id:93
I161111 17:23:33.971584 5433 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:92}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:33.973556 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3eb7511c at index 800 in 58.566µs.
I161111 17:23:34.098637 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 993, log entries: 46
I161111 17:23:34.102764 452 storage/store.go:2988  [s3,r1/92:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:34.178082 340 storage/store.go:2041  [s3,r1/92:/M{in-ax}] DEL 1
I161111 17:23:34.178633 5408 storage/store.go:1912  [s3] ADD 1
I161111 17:23:34.178713 5408 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:34.180684 5408 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:34.180703 5408 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:34.180726 5408 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:34.180738 5408 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:34.182291 5408 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 800, term: 6]
I161111 17:23:34.182314 5408 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 800, term: 6]
I161111 17:23:34.182335 5408 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 801, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.182352 5408 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 801, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.182364 5408 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 800] restored snapshot [index: 800, term: 6]
I161111 17:23:34.182384 5408 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 800 (id=3eb7511c, encoded size=16, 1 rocksdb batches, 46 log entries)
I161111 17:23:34.215652 5408 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.033s
I161111 17:23:34.244052 5408 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:34.245072 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:93
I161111 17:23:34.246591 5461 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:93}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:93}]
I161111 17:23:34.299621 267 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5d became follower at term 6
I161111 17:23:34.299662 267 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5d [peers: [1,2], term: 6, commit: 800, applied: 800, lastindex: 800, lastterm: 6]
I161111 17:23:34.299704 267 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5d [logterm: 0, index: 805] rejected msgApp [logterm: 6, index: 805] from 1
I161111 17:23:34.299879 100 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 800) from 5d for index 805
I161111 17:23:34.299901 100 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5d to [next = 801, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.324405 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:93 > next_replica_id:94
I161111 17:23:34.334083 5421 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:93}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:34.335942 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot cacad23d at index 810 in 52.905µs.
I161111 17:23:34.341064 452 storage/store.go:2988  [s3,r1/93:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:34.343294 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1005, log entries: 56
I161111 17:23:34.385666 340 storage/store.go:2041  [s3,r1/93:/M{in-ax}] DEL 1
I161111 17:23:34.386203 5522 storage/store.go:1912  [s3] ADD 1
I161111 17:23:34.386364 5522 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:34.388993 5522 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:34.389049 5522 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:34.389087 5522 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:34.389131 5522 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:34.391313 5522 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 810, term: 6]
I161111 17:23:34.391347 5522 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 810, term: 6]
I161111 17:23:34.391381 5522 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 811, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.391409 5522 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 811, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.391429 5522 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 810] restored snapshot [index: 810, term: 6]
I161111 17:23:34.391454 5522 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 810 (id=cacad23d, encoded size=16, 1 rocksdb batches, 56 log entries)
I161111 17:23:34.675059 5522 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.284s
I161111 17:23:34.675091 5522 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:34.675281 286 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5d became follower at term 6
I161111 17:23:34.675343 286 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5d [peers: [1,2], term: 6, commit: 810, applied: 810, lastindex: 810, lastterm: 6]
I161111 17:23:34.688517 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:94
I161111 17:23:34.695492 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:34.696332 5532 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:94}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:94}]
I161111 17:23:34.754841 340 storage/store.go:2041  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:34.755836 245 raft/raft.go:568  [s3,r1/94:{-}] 5e became follower at term 0
I161111 17:23:34.757979 245 raft/raft.go:316  [s3,r1/94:{-}] newRaft 5e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:34.757996 245 raft/raft.go:568  [s3,r1/94:{-}] 5e became follower at term 1
I161111 17:23:34.758018 245 raft/raft.go:710  [s3,r1/94:{-}] 5e [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:34.758029 245 raft/raft.go:568  [s3,r1/94:{-}] 5e became follower at term 6
I161111 17:23:34.787694 243 raft/raft.go:1088  [s3,r1/94:{-}] 5e [logterm: 0, index: 818] rejected msgApp [logterm: 6, index: 818] from 1
I161111 17:23:34.787843 90 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 5e for index 818
I161111 17:23:34.787871 90 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5e to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.788011 90 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 17a65bd6 at index 818 in 89.888µs.
I161111 17:23:34.788050 90 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 755, commit: 818] sent snapshot[index: 818, term: 6] to 5e [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.788067 90 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 5e [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 818]
I161111 17:23:34.790555 5515 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1014, log entries: 64
I161111 17:23:34.790679 5518 storage/store.go:1912  [s3] ADD 1
I161111 17:23:34.793401 5518 raft/raft.go:1125  [s3,r1/94:{-}] 5e [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 818, term: 6]
I161111 17:23:34.793436 5518 raft/log.go:288  [s3,r1/94:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 818, term: 6]
I161111 17:23:34.793484 5518 raft/raft.go:1135  [s3,r1/94:{-}] 5e restored progress of 1 [next = 819, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.793513 5518 raft/raft.go:1135  [s3,r1/94:{-}] 5e restored progress of 2 [next = 819, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.793555 5518 raft/raft.go:1135  [s3,r1/94:{-}] 5e restored progress of 5e [next = 819, match = 818, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:34.793579 5518 raft/raft.go:1102  [s3,r1/94:{-}] 5e [commit: 818] restored snapshot [index: 818, term: 6]
I161111 17:23:34.793687 5518 storage/replica_raftstorage.go:589  [s3,r1/94:{-}] applying Raft snapshot at index 818 (id=17a65bd6, encoded size=16, 1 rocksdb batches, 64 log entries)
I161111 17:23:35.069185 5518 storage/replica_raftstorage.go:592  [s3,r1/94:/M{in-ax}] applied Raft snapshot in 0.275s
I161111 17:23:35.070099 70 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 5e [next = 819, match = 818, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 818]
I161111 17:23:35.245146 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:94 > next_replica_id:95
I161111 17:23:35.247028 5576 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:94}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:35.249609 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 36e6376c at index 829 in 54.796µs.
I161111 17:23:35.250338 452 storage/store.go:2988  [s3,r1/94:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:35.258523 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1026, log entries: 75
I161111 17:23:35.296155 340 storage/store.go:2041  [s3,r1/94:/M{in-ax}] DEL 1
I161111 17:23:35.296841 5609 storage/store.go:1912  [s3] ADD 1
I161111 17:23:35.296950 5609 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:35.299115 5609 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:35.299147 5609 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:35.299182 5609 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:35.299202 5609 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
W161111 17:23:35.300306 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 94}: raft group deleted
I161111 17:23:35.301300 5609 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 829, term: 6]
I161111 17:23:35.301366 5609 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 829, term: 6]
I161111 17:23:35.301420 5609 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 830, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.301463 5609 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 830, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.301498 5609 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 829] restored snapshot [index: 829, term: 6]
I161111 17:23:35.301545 5609 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 829 (id=36e6376c, encoded size=16, 1 rocksdb batches, 75 log entries)
I161111 17:23:35.426587 5609 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.125s
I161111 17:23:35.426628 5609 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:35.427429 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:95
I161111 17:23:35.429028 5650 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:95}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:95}]
I161111 17:23:35.553684 275 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 5f became follower at term 6
I161111 17:23:35.553730 275 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 5f [peers: [1,2], term: 6, commit: 829, applied: 829, lastindex: 829, lastterm: 6]
I161111 17:23:35.560203 278 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 5f [logterm: 0, index: 834] rejected msgApp [logterm: 6, index: 834] from 1
I161111 17:23:35.560376 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 829) from 5f for index 834
I161111 17:23:35.560400 91 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 5f to [next = 830, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.599290 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:95 > next_replica_id:96
I161111 17:23:35.602645 5659 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:95}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:35.604441 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 2ffff84f at index 839 in 51.4µs.
I161111 17:23:35.610252 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1038, log entries: 85
I161111 17:23:35.610479 452 storage/store.go:2988  [s3,r1/95:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:35.763857 340 storage/store.go:2041  [s3,r1/95:/M{in-ax}] DEL 1
I161111 17:23:35.764379 5590 storage/store.go:1912  [s3] ADD 1
I161111 17:23:35.764484 5590 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:35.766896 5590 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:35.766930 5590 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:35.766965 5590 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:35.766990 5590 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:35.773916 5590 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 839, term: 6]
I161111 17:23:35.773959 5590 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 839, term: 6]
I161111 17:23:35.773993 5590 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 840, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.774024 5590 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 840, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.774049 5590 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 839] restored snapshot [index: 839, term: 6]
I161111 17:23:35.776661 5590 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 839 (id=2ffff84f, encoded size=16, 1 rocksdb batches, 85 log entries)
I161111 17:23:35.898709 5590 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.122s
I161111 17:23:35.898769 5590 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:35.899839 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:96
I161111 17:23:35.901839 5620 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:96}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:96}]
I161111 17:23:35.921160 250 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 60 became follower at term 6
I161111 17:23:35.921204 250 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 60 [peers: [1,2], term: 6, commit: 839, applied: 839, lastindex: 839, lastterm: 6]
I161111 17:23:35.921251 250 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 60 [logterm: 0, index: 844] rejected msgApp [logterm: 6, index: 844] from 1
I161111 17:23:35.921435 69 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 839) from 60 for index 844
I161111 17:23:35.921460 69 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 60 to [next = 840, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:35.925134 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:96 > next_replica_id:97
I161111 17:23:35.927044 5749 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:96}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:35.951230 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 556bca68 at index 847 in 53.856µs.
I161111 17:23:35.954256 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1048, log entries: 93
I161111 17:23:35.954464 5756 raft/raft.go:568  [s3,r1/96:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:35.954504 5756 raft/raft.go:316  [s3,r1/96:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 847, applied: 847, lastindex: 847, lastterm: 6]
I161111 17:23:35.954559 5756 raft/raft.go:1106  [s3,r1/96:/M{in-ax}] ffffffffffffffff [commit: 847] ignored snapshot [index: 847, term: 6]
I161111 17:23:35.955097 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:97
I161111 17:23:35.960509 452 storage/store.go:2988  [s3,r1/96:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:35.974670 5629 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:97}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:97}]
I161111 17:23:36.039803 340 storage/store.go:2041  [s3,r1/96:/M{in-ax}] DEL 1
I161111 17:23:36.054040 263 raft/raft.go:568  [s3,r1/97:{-}] 61 became follower at term 0
I161111 17:23:36.056677 263 raft/raft.go:316  [s3,r1/97:{-}] newRaft 61 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:36.056698 263 raft/raft.go:568  [s3,r1/97:{-}] 61 became follower at term 1
I161111 17:23:36.056721 263 raft/raft.go:710  [s3,r1/97:{-}] 61 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:36.056734 263 raft/raft.go:568  [s3,r1/97:{-}] 61 became follower at term 6
I161111 17:23:36.087492 267 raft/raft.go:1088  [s3,r1/97:{-}] 61 [logterm: 0, index: 851] rejected msgApp [logterm: 6, index: 851] from 1
I161111 17:23:36.087668 104 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 61 for index 851
I161111 17:23:36.087695 104 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 61 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.087831 104 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 39546b9d at index 851 in 83.052µs.
I161111 17:23:36.087867 104 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 755, commit: 851] sent snapshot[index: 851, term: 6] to 61 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.087885 104 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 61 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 851]
I161111 17:23:36.091226 5766 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1053, log entries: 97
I161111 17:23:36.091458 5769 storage/store.go:1912  [s3] ADD 1
I161111 17:23:36.140504 5769 raft/raft.go:1125  [s3,r1/97:{-}] 61 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 851, term: 6]
I161111 17:23:36.140573 5769 raft/log.go:288  [s3,r1/97:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 851, term: 6]
I161111 17:23:36.140604 5769 raft/raft.go:1135  [s3,r1/97:{-}] 61 restored progress of 1 [next = 852, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.140652 5769 raft/raft.go:1135  [s3,r1/97:{-}] 61 restored progress of 2 [next = 852, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.140677 5769 raft/raft.go:1135  [s3,r1/97:{-}] 61 restored progress of 61 [next = 852, match = 851, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.140721 5769 raft/raft.go:1102  [s3,r1/97:{-}] 61 [commit: 851] restored snapshot [index: 851, term: 6]
I161111 17:23:36.140765 5769 storage/replica_raftstorage.go:589  [s3,r1/97:{-}] applying Raft snapshot at index 851 (id=39546b9d, encoded size=16, 1 rocksdb batches, 97 log entries)
I161111 17:23:36.315596 5769 storage/replica_raftstorage.go:592  [s3,r1/97:/M{in-ax}] applied Raft snapshot in 0.175s
I161111 17:23:36.316182 71 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 61 [next = 852, match = 851, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 851]
I161111 17:23:36.477057 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 755-859
I161111 17:23:36.547615 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:97 > next_replica_id:98
I161111 17:23:36.549558 5784 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:97}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:36.566459 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 38ed1257 at index 863 in 55.419µs.
I161111 17:23:36.591601 452 storage/store.go:2988  [s3,r1/97:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:36.593756 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1065, log entries: 5
I161111 17:23:36.756212 340 storage/store.go:2041  [s3,r1/97:/M{in-ax}] DEL 1
I161111 17:23:36.759669 5829 storage/store.go:1912  [s3] ADD 1
I161111 17:23:36.759761 5829 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:36.762118 5829 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:36.762139 5829 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:36.762161 5829 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:36.762174 5829 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:36.764099 5829 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 863, term: 6]
I161111 17:23:36.764197 5829 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 863, term: 6]
I161111 17:23:36.764236 5829 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 864, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.764283 5829 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 864, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:36.764307 5829 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 863] restored snapshot [index: 863, term: 6]
I161111 17:23:36.764353 5829 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 863 (id=38ed1257, encoded size=16, 1 rocksdb batches, 5 log entries)
I161111 17:23:36.920278 5829 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.156s
I161111 17:23:36.920320 5829 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:36.921215 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:98
I161111 17:23:36.922884 5874 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:98}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:98}]
I161111 17:23:37.013866 254 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 62 became follower at term 6
I161111 17:23:37.013918 254 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 62 [peers: [1,2], term: 6, commit: 863, applied: 863, lastindex: 863, lastterm: 6]
I161111 17:23:37.013967 254 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 62 [logterm: 0, index: 871] rejected msgApp [logterm: 6, index: 871] from 1
I161111 17:23:37.014160 68 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 863) from 62 for index 871
I161111 17:23:37.014187 68 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 62 to [next = 864, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.035987 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:98 > next_replica_id:99
I161111 17:23:37.038323 5741 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:98}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:37.061127 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7df70efc at index 876 in 55.933µs.
I161111 17:23:37.066424 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1080, log entries: 18
I161111 17:23:37.085588 452 storage/store.go:2988  [s3,r1/98:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:37.137337 340 storage/store.go:2041  [s3,r1/98:/M{in-ax}] DEL 1
I161111 17:23:37.150942 5821 storage/store.go:1912  [s3] ADD 1
I161111 17:23:37.151073 5821 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:37.157264 5821 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:37.157299 5821 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:37.157329 5821 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:37.157349 5821 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:37.162547 5821 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 876, term: 6]
I161111 17:23:37.162608 5821 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 876, term: 6]
I161111 17:23:37.162640 5821 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 877, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.162665 5821 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 877, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.162702 5821 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 876] restored snapshot [index: 876, term: 6]
I161111 17:23:37.162730 5821 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 876 (id=7df70efc, encoded size=16, 1 rocksdb batches, 18 log entries)
I161111 17:23:37.345485 5821 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.183s
I161111 17:23:37.345633 5821 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:37.346727 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:99
I161111 17:23:37.349007 5920 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:99}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:99}]
I161111 17:23:37.358341 275 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 63 became follower at term 6
I161111 17:23:37.358379 275 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 63 [peers: [1,2], term: 6, commit: 876, applied: 876, lastindex: 876, lastterm: 6]
I161111 17:23:37.358422 275 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 63 [logterm: 0, index: 881] rejected msgApp [logterm: 6, index: 881] from 1
I161111 17:23:37.358589 105 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 876) from 63 for index 881
I161111 17:23:37.358612 105 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 63 to [next = 877, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.379579 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:99 > next_replica_id:100
I161111 17:23:37.382855 5902 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:99}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:37.395595 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot f765a3b3 at index 886 in 71.294µs.
I161111 17:23:37.430694 452 storage/store.go:2988  [s3,r1/99:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:37.437373 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1092, log entries: 28
I161111 17:23:37.451364 340 storage/store.go:2041  [s3,r1/99:/M{in-ax}] DEL 1
I161111 17:23:37.452336 5935 storage/store.go:1912  [s3] ADD 1
I161111 17:23:37.452488 5935 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:37.463171 5935 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:37.463229 5935 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:37.463267 5935 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:37.463315 5935 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:37.469095 5935 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 886, term: 6]
I161111 17:23:37.469136 5935 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 886, term: 6]
I161111 17:23:37.469348 5935 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 887, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.469375 5935 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 887, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.469502 5935 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 886] restored snapshot [index: 886, term: 6]
I161111 17:23:37.469592 5935 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 886 (id=f765a3b3, encoded size=16, 1 rocksdb batches, 28 log entries)
I161111 17:23:37.811119 5935 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.341s
I161111 17:23:37.811355 5935 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:37.812505 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:100
I161111 17:23:37.814073 5980 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:100}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:100}]
I161111 17:23:37.907011 285 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 64 became follower at term 6
I161111 17:23:37.907054 285 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 64 [peers: [1,2], term: 6, commit: 886, applied: 886, lastindex: 886, lastterm: 6]
I161111 17:23:37.907101 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 64 [logterm: 0, index: 894] rejected msgApp [logterm: 6, index: 894] from 1
I161111 17:23:37.907262 85 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 886) from 64 for index 894
I161111 17:23:37.907288 85 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 64 to [next = 887, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:37.973243 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:100 > next_replica_id:101
I161111 17:23:37.975072 6035 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:100}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:37.977614 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot efcb1bd4 at index 899 in 70.731µs.
I161111 17:23:37.978256 452 storage/store.go:2988  [s3,r1/100:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:37.986947 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1107, log entries: 41
I161111 17:23:37.991736 452 storage/store.go:2988  [s3,r1/100:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:38.007347 340 storage/store.go:2041  [s3,r1/100:/M{in-ax}] DEL 1
I161111 17:23:38.022587 5992 storage/store.go:1912  [s3] ADD 1
I161111 17:23:38.022671 5992 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:38.025592 5992 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:38.025627 5992 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:38.025663 5992 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:38.025688 5992 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:38.027931 5992 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 899, term: 6]
I161111 17:23:38.027968 5992 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 899, term: 6]
I161111 17:23:38.028002 5992 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 900, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:38.028032 5992 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 900, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:38.028057 5992 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 899] restored snapshot [index: 899, term: 6]
I161111 17:23:38.028085 5992 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 899 (id=efcb1bd4, encoded size=16, 1 rocksdb batches, 41 log entries)
W161111 17:23:38.047603 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 100}: raft group deleted
I161111 17:23:38.306515 5992 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.278s
I161111 17:23:38.306572 5992 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:38.307359 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:101
I161111 17:23:38.308934 6067 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:101}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:101}]
I161111 17:23:38.438691 264 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 65 became follower at term 6
I161111 17:23:38.438742 264 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 65 [peers: [1,2], term: 6, commit: 899, applied: 899, lastindex: 899, lastterm: 6]
I161111 17:23:38.438795 264 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 65 [logterm: 0, index: 907] rejected msgApp [logterm: 6, index: 907] from 1
I161111 17:23:38.438990 74 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 899) from 65 for index 907
I161111 17:23:38.439024 74 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 65 to [next = 900, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:38.472521 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:101 > next_replica_id:102
I161111 17:23:38.474477 6115 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:101}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:38.479959 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ea9f2168 at index 912 in 100.5µs.
I161111 17:23:38.483130 452 storage/store.go:2988  [s3,r1/101:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:38.486324 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1122, log entries: 54
I161111 17:23:38.651288 14 gossip/gossip.go:1071  [n1] node has connected to cluster via gossip
I161111 17:23:38.714419 340 storage/store.go:2041  [s3,r1/101:/M{in-ax}] DEL 1
I161111 17:23:38.742098 6137 storage/store.go:1912  [s3] ADD 1
I161111 17:23:38.742792 6137 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:38.753440 6137 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:38.753796 6137 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:38.753832 6137 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:38.754894 6137 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:38.764474 6137 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 912, term: 6]
I161111 17:23:38.792564 6137 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 912, term: 6]
I161111 17:23:38.792642 6137 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 913, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:38.792717 6137 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 913, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:38.792880 6137 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 912] restored snapshot [index: 912, term: 6]
I161111 17:23:38.792912 6137 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 912 (id=ea9f2168, encoded size=16, 1 rocksdb batches, 54 log entries)
I161111 17:23:39.024105 6137 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.231s
I161111 17:23:39.024337 6137 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:39.025246 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:102
I161111 17:23:39.026948 6110 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:102}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:102}]
I161111 17:23:39.153675 287 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 66 became follower at term 6
I161111 17:23:39.153726 287 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 66 [peers: [1,2], term: 6, commit: 912, applied: 912, lastindex: 912, lastterm: 6]
I161111 17:23:39.160557 285 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 66 [logterm: 0, index: 923] rejected msgApp [logterm: 6, index: 923] from 1
I161111 17:23:39.160731 68 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 912) from 66 for index 923
I161111 17:23:39.160758 68 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 66 to [next = 913, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.302905 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:102 > next_replica_id:103
I161111 17:23:39.304954 6140 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:102}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:39.307449 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a2f02ee2 at index 928 in 54.779µs.
I161111 17:23:39.314663 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1140, log entries: 70
I161111 17:23:39.315617 452 storage/store.go:2988  [s3,r1/102:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:39.457653 340 storage/store.go:2041  [s3,r1/102:/M{in-ax}] DEL 1
I161111 17:23:39.458611 6168 storage/store.go:1912  [s3] ADD 1
I161111 17:23:39.458704 6168 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:39.461274 6168 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:39.461294 6168 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:39.461314 6168 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:39.461325 6168 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:39.463370 6168 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 928, term: 6]
I161111 17:23:39.463388 6168 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 928, term: 6]
I161111 17:23:39.463409 6168 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 929, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.463424 6168 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 929, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.463435 6168 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 928] restored snapshot [index: 928, term: 6]
I161111 17:23:39.463452 6168 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 928 (id=a2f02ee2, encoded size=16, 1 rocksdb batches, 70 log entries)
I161111 17:23:39.799933 6168 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.336s
I161111 17:23:39.800317 6168 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:39.800477 263 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 66 became follower at term 6
I161111 17:23:39.800521 263 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 66 [peers: [1,2], term: 6, commit: 928, applied: 928, lastindex: 928, lastterm: 6]
I161111 17:23:39.801701 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:103
I161111 17:23:39.803149 6267 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:103}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:103}]
I161111 17:23:39.817117 260 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 67 became follower at term 6
I161111 17:23:39.817188 260 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 67 [peers: [1,2], term: 6, commit: 928, applied: 928, lastindex: 928, lastterm: 6]
I161111 17:23:39.838342 270 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 67 [logterm: 0, index: 939] rejected msgApp [logterm: 6, index: 939] from 1
I161111 17:23:39.838473 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:39.839255 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 928) from 67 for index 939
I161111 17:23:39.839290 91 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 67 to [next = 929, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.843804 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:103 > next_replica_id:104
I161111 17:23:39.846323 6254 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:103}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:39.849850 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9b824505 at index 941 in 89.567µs.
I161111 17:23:39.850752 452 storage/store.go:2988  [s3,r1/103:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:39.858917 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1155, log entries: 83
I161111 17:23:39.920336 340 storage/store.go:2041  [s3,r1/103:/M{in-ax}] DEL 1
I161111 17:23:39.921038 6237 storage/store.go:1912  [s3] ADD 1
I161111 17:23:39.921127 6237 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:39.936012 6237 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:39.936084 6237 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:39.936162 6237 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:39.936189 6237 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:39.940511 6237 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 941, term: 6]
I161111 17:23:39.940561 6237 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 941, term: 6]
I161111 17:23:39.940617 6237 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 942, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.940649 6237 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 942, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:39.940693 6237 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 941] restored snapshot [index: 941, term: 6]
I161111 17:23:39.940736 6237 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 941 (id=9b824505, encoded size=16, 1 rocksdb batches, 83 log entries)
W161111 17:23:39.940908 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 103}: raft group deleted
I161111 17:23:40.096193 6237 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.155s
I161111 17:23:40.096412 6237 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:40.097493 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:104
I161111 17:23:40.102653 6222 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:104}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:104}]
I161111 17:23:40.104721 281 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 68 became follower at term 6
I161111 17:23:40.104764 281 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 68 [peers: [1,2], term: 6, commit: 941, applied: 941, lastindex: 941, lastterm: 6]
I161111 17:23:40.137456 286 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 68 [logterm: 0, index: 949] rejected msgApp [logterm: 6, index: 949] from 1
I161111 17:23:40.137611 112 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 941) from 68 for index 949
I161111 17:23:40.137638 112 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 68 to [next = 942, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:40.148877 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:104 > next_replica_id:105
I161111 17:23:40.150931 6209 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:104}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:40.156627 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot b9f09a25 at index 951 in 58.364µs.
I161111 17:23:40.157460 452 storage/store.go:2988  [s3,r1/104:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:40.168908 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1167, log entries: 93
I161111 17:23:40.326902 340 storage/store.go:2041  [s3,r1/104:/M{in-ax}] DEL 1
I161111 17:23:40.329653 6365 storage/store.go:1912  [s3] ADD 1
I161111 17:23:40.329744 6365 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:40.332497 6365 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:40.332517 6365 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:40.332550 6365 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:40.332564 6365 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:40.336878 6365 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 951, term: 6]
I161111 17:23:40.336920 6365 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 951, term: 6]
I161111 17:23:40.336977 6365 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 952, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:40.337005 6365 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 952, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:40.337091 6365 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 951] restored snapshot [index: 951, term: 6]
I161111 17:23:40.337162 6365 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 951 (id=b9f09a25, encoded size=16, 1 rocksdb batches, 93 log entries)
I161111 17:23:40.626220 6365 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.289s
I161111 17:23:40.626470 6365 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:40.627557 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:105
I161111 17:23:40.630088 6383 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:105}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:105}]
I161111 17:23:40.753682 255 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 69 became follower at term 6
I161111 17:23:40.753730 255 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 69 [peers: [1,2], term: 6, commit: 951, applied: 951, lastindex: 951, lastterm: 6]
I161111 17:23:40.781959 253 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 69 [logterm: 0, index: 962] rejected msgApp [logterm: 6, index: 962] from 1
I161111 17:23:40.782128 106 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 951) from 69 for index 962
I161111 17:23:40.782153 106 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 69 to [next = 952, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:40.783072 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 859-963
I161111 17:23:40.796478 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:105 > next_replica_id:106
I161111 17:23:40.884815 6408 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:105}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:40.887108 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot e3cb69f9 at index 968 in 62.298µs.
I161111 17:23:40.887441 452 storage/store.go:2988  [s3,r1/105:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:40.942506 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1185, log entries: 6
I161111 17:23:41.088010 340 storage/store.go:2041  [s3,r1/105:/M{in-ax}] DEL 1
I161111 17:23:41.088763 6411 storage/store.go:1912  [s3] ADD 1
I161111 17:23:41.088840 6411 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:41.091707 6411 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:41.091846 6411 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
W161111 17:23:41.091912 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 105}: raft group deleted
I161111 17:23:41.091946 6411 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:41.091968 6411 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:41.095828 6411 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 968, term: 6]
I161111 17:23:41.095867 6411 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 968, term: 6]
I161111 17:23:41.095925 6411 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 969, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:41.095958 6411 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 969, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:41.096001 6411 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 968] restored snapshot [index: 968, term: 6]
I161111 17:23:41.096033 6411 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 968 (id=e3cb69f9, encoded size=16, 1 rocksdb batches, 6 log entries)
I161111 17:23:41.182063 6411 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.086s
I161111 17:23:41.182237 6411 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:41.183660 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:106
I161111 17:23:41.185263 6468 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:106}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:106}]
I161111 17:23:41.254467 277 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6a became follower at term 6
I161111 17:23:41.254513 277 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6a [peers: [1,2], term: 6, commit: 968, applied: 968, lastindex: 968, lastterm: 6]
I161111 17:23:41.287509 284 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6a [logterm: 0, index: 976] rejected msgApp [logterm: 6, index: 976] from 1
I161111 17:23:41.287659 70 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 968) from 6a for index 976
I161111 17:23:41.287687 70 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6a to [next = 969, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:41.339981 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:106 > next_replica_id:107
I161111 17:23:41.348910 6447 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:106}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:41.357663 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot f31db1b1 at index 981 in 48.356µs.
I161111 17:23:41.366729 452 storage/store.go:2988  [s3,r1/106:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:41.372020 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1200, log entries: 19
I161111 17:23:41.471474 340 storage/store.go:2041  [s3,r1/106:/M{in-ax}] DEL 1
I161111 17:23:41.472066 6336 storage/store.go:1912  [s3] ADD 1
I161111 17:23:41.472242 6336 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:41.591786 6336 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:41.592037 6336 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:41.592214 6336 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:41.592619 6336 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:41.616170 6336 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 981, term: 6]
I161111 17:23:41.617145 6336 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 981, term: 6]
I161111 17:23:41.617379 6336 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 982, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:41.617621 6336 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 982, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:41.617653 6336 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 981] restored snapshot [index: 981, term: 6]
I161111 17:23:41.617891 6336 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 981 (id=f31db1b1, encoded size=16, 1 rocksdb batches, 19 log entries)
I161111 17:23:41.820328 160 gossip/gossip.go:1071  [n2] node has connected to cluster via gossip
I161111 17:23:41.880003 6336 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.262s
I161111 17:23:41.882246 6336 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:41.883182 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:107
I161111 17:23:41.889428 6508 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:107}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:107}]
I161111 17:23:41.953711 252 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6b became follower at term 6
I161111 17:23:41.953757 252 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6b [peers: [1,2], term: 6, commit: 981, applied: 981, lastindex: 981, lastterm: 6]
I161111 17:23:41.992515 256 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6b [logterm: 0, index: 992] rejected msgApp [logterm: 6, index: 992] from 1
I161111 17:23:41.992682 76 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 981) from 6b for index 992
I161111 17:23:41.992713 76 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6b to [next = 982, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.042232 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:107 > next_replica_id:108
I161111 17:23:42.045001 6432 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:107}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:42.048418 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a3deba64 at index 995 in 50.535µs.
I161111 17:23:42.049755 452 storage/store.go:2988  [s3,r1/107:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:42.054621 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1216, log entries: 33
I161111 17:23:42.160504 340 storage/store.go:2041  [s3,r1/107:/M{in-ax}] DEL 1
I161111 17:23:42.161046 6497 storage/store.go:1912  [s3] ADD 1
I161111 17:23:42.161127 6497 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:42.163952 6497 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:42.163977 6497 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:42.163999 6497 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:42.164010 6497 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
W161111 17:23:42.164235 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 107}: raft group deleted
I161111 17:23:42.171554 6497 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 995, term: 6]
I161111 17:23:42.171622 6497 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 995, term: 6]
I161111 17:23:42.171657 6497 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 996, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.171708 6497 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 996, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.171730 6497 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 995] restored snapshot [index: 995, term: 6]
I161111 17:23:42.171791 6497 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 995 (id=a3deba64, encoded size=16, 1 rocksdb batches, 33 log entries)
I161111 17:23:42.373019 6497 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.201s
I161111 17:23:42.373076 6497 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:42.373842 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:108
I161111 17:23:42.375379 6633 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:108}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:108}]
I161111 17:23:42.398500 274 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6c became follower at term 6
I161111 17:23:42.398544 274 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6c [peers: [1,2], term: 6, commit: 995, applied: 995, lastindex: 995, lastterm: 6]
I161111 17:23:42.398591 274 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6c [logterm: 0, index: 1002] rejected msgApp [logterm: 6, index: 1002] from 1
I161111 17:23:42.399543 85 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 995) from 6c for index 1002
I161111 17:23:42.399566 85 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6c to [next = 996, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.437839 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:108 > next_replica_id:109
I161111 17:23:42.439408 6657 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:108}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:42.441962 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 5b4afdb3 at index 1007 in 49.194µs.
I161111 17:23:42.442674 452 storage/store.go:2988  [s3,r1/108:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:42.450651 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1230, log entries: 45
I161111 17:23:42.517062 340 storage/store.go:2041  [s3,r1/108:/M{in-ax}] DEL 1
I161111 17:23:42.517628 6616 storage/store.go:1912  [s3] ADD 1
I161111 17:23:42.517714 6616 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:42.520668 6616 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:42.520689 6616 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:42.520713 6616 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:42.520726 6616 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:42.525183 6616 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1007, term: 6]
I161111 17:23:42.525237 6616 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1007, term: 6]
I161111 17:23:42.525272 6616 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1008, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.525296 6616 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1008, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.525333 6616 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1007] restored snapshot [index: 1007, term: 6]
I161111 17:23:42.525361 6616 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1007 (id=5b4afdb3, encoded size=16, 1 rocksdb batches, 45 log entries)
W161111 17:23:42.532296 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 108}: raft group deleted
I161111 17:23:42.560383 6616 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.035s
I161111 17:23:42.560429 6616 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:42.561268 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:109
I161111 17:23:42.562848 6576 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:109}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:109}]
I161111 17:23:42.578575 283 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6d became follower at term 6
I161111 17:23:42.578620 283 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6d [peers: [1,2], term: 6, commit: 1007, applied: 1007, lastindex: 1007, lastterm: 6]
I161111 17:23:42.578667 283 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6d [logterm: 0, index: 1009] rejected msgApp [logterm: 6, index: 1009] from 1
I161111 17:23:42.578855 92 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1007) from 6d for index 1009
I161111 17:23:42.578882 92 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6d to [next = 1008, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.599768 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:109 > next_replica_id:110
I161111 17:23:42.601573 6671 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:109}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:42.616353 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot e6ee56cd at index 1012 in 54.404µs.
I161111 17:23:42.617396 452 storage/store.go:2988  [s3,r1/109:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:42.619813 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1237, log entries: 50
I161111 17:23:42.620083 6590 raft/raft.go:568  [s3,r1/109:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:42.620134 6590 raft/raft.go:316  [s3,r1/109:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 1012, applied: 1012, lastindex: 1012, lastterm: 6]
I161111 17:23:42.620181 6590 raft/raft.go:1106  [s3,r1/109:/M{in-ax}] ffffffffffffffff [commit: 1012] ignored snapshot [index: 1012, term: 6]
I161111 17:23:42.620794 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:110
I161111 17:23:42.626162 6699 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:110}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:110}]
I161111 17:23:42.726855 340 storage/store.go:2041  [s3,r1/109:/M{in-ax}] DEL 1
I161111 17:23:42.753919 247 raft/raft.go:568  [s3,r1/110:{-}] 6e became follower at term 0
I161111 17:23:42.757027 247 raft/raft.go:316  [s3,r1/110:{-}] newRaft 6e [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:42.757048 247 raft/raft.go:568  [s3,r1/110:{-}] 6e became follower at term 1
I161111 17:23:42.757072 247 raft/raft.go:710  [s3,r1/110:{-}] 6e [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:42.757083 247 raft/raft.go:568  [s3,r1/110:{-}] 6e became follower at term 6
I161111 17:23:42.760915 250 raft/raft.go:1088  [s3,r1/110:{-}] 6e [logterm: 0, index: 1016] rejected msgApp [logterm: 6, index: 1016] from 1
I161111 17:23:42.765468 16 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 6e for index 1016
I161111 17:23:42.765561 16 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6e to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.765785 16 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot c63f1af6 at index 1017 in 93.217µs.
I161111 17:23:42.765865 16 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 963, commit: 1017] sent snapshot[index: 1017, term: 6] to 6e [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.765897 16 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 6e [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1017]
I161111 17:23:42.827146 6601 storage/store.go:1912  [s3] ADD 1
I161111 17:23:42.830442 6601 raft/raft.go:1125  [s3,r1/110:{-}] 6e [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1017, term: 6]
I161111 17:23:42.830483 6601 raft/log.go:288  [s3,r1/110:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1017, term: 6]
I161111 17:23:42.830516 6601 raft/raft.go:1135  [s3,r1/110:{-}] 6e restored progress of 1 [next = 1018, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.830539 6601 raft/raft.go:1135  [s3,r1/110:{-}] 6e restored progress of 2 [next = 1018, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.830562 6601 raft/raft.go:1135  [s3,r1/110:{-}] 6e restored progress of 6e [next = 1018, match = 1017, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:42.830582 6601 raft/raft.go:1102  [s3,r1/110:{-}] 6e [commit: 1017] restored snapshot [index: 1017, term: 6]
I161111 17:23:42.830617 6601 storage/replica_raftstorage.go:589  [s3,r1/110:{-}] applying Raft snapshot at index 1017 (id=c63f1af6, encoded size=16, 1 rocksdb batches, 55 log entries)
I161111 17:23:42.848948 6701 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1243, log entries: 55
I161111 17:23:43.006668 6601 storage/replica_raftstorage.go:592  [s3,r1/110:/M{in-ax}] applied Raft snapshot in 0.176s
I161111 17:23:43.007074 96 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 6e [next = 1018, match = 1017, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1017]
I161111 17:23:43.265529 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:110 > next_replica_id:111
I161111 17:23:43.267392 6556 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:110}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:43.270025 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot bc41df9f at index 1027 in 62.715µs.
I161111 17:23:43.270754 452 storage/store.go:2988  [s3,r1/110:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:43.277384 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1254, log entries: 65
I161111 17:23:43.431232 340 storage/store.go:2041  [s3,r1/110:/M{in-ax}] DEL 1
I161111 17:23:43.432103 6718 storage/store.go:1912  [s3] ADD 1
I161111 17:23:43.432204 6718 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
W161111 17:23:43.432705 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 110}: raft group deleted
I161111 17:23:43.436426 6718 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:43.436460 6718 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:43.436494 6718 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:43.436534 6718 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:43.439530 6718 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1027, term: 6]
I161111 17:23:43.439564 6718 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1027, term: 6]
I161111 17:23:43.439593 6718 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1028, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:43.439617 6718 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1028, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:43.439638 6718 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1027] restored snapshot [index: 1027, term: 6]
I161111 17:23:43.439666 6718 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1027 (id=bc41df9f, encoded size=16, 1 rocksdb batches, 65 log entries)
I161111 17:23:43.737978 6718 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.298s
I161111 17:23:43.738016 6718 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:43.738754 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:111
I161111 17:23:43.740253 6834 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:111}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:111}]
I161111 17:23:43.840527 287 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 6f became follower at term 6
I161111 17:23:43.840572 287 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 6f [peers: [1,2], term: 6, commit: 1027, applied: 1027, lastindex: 1027, lastterm: 6]
I161111 17:23:43.840620 287 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 6f [logterm: 0, index: 1038] rejected msgApp [logterm: 6, index: 1038] from 1
I161111 17:23:43.840784 66 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1027) from 6f for index 1038
I161111 17:23:43.840809 66 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 6f to [next = 1028, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.021212 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:111 > next_replica_id:112
I161111 17:23:44.024734 6767 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:111}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:44.027761 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot db6d0348 at index 1044 in 50.842µs.
I161111 17:23:44.028589 452 storage/store.go:2988  [s3,r1/111:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:44.031720 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1273, log entries: 82
I161111 17:23:44.135577 340 storage/store.go:2041  [s3,r1/111:/M{in-ax}] DEL 1
I161111 17:23:44.136066 6849 storage/store.go:1912  [s3] ADD 1
I161111 17:23:44.136187 6849 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:44.139339 6849 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:44.139359 6849 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:44.139382 6849 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:44.139393 6849 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:44.142207 6849 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1044, term: 6]
I161111 17:23:44.142247 6849 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1044, term: 6]
I161111 17:23:44.142282 6849 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1045, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.142312 6849 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1045, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.142336 6849 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1044] restored snapshot [index: 1044, term: 6]
I161111 17:23:44.142363 6849 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1044 (id=db6d0348, encoded size=16, 1 rocksdb batches, 82 log entries)
I161111 17:23:44.288591 6849 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.146s
I161111 17:23:44.288633 6849 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:44.289429 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:112
I161111 17:23:44.290877 6890 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:112}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:112}]
I161111 17:23:44.306057 258 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 70 became follower at term 6
I161111 17:23:44.306100 258 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 70 [peers: [1,2], term: 6, commit: 1044, applied: 1044, lastindex: 1044, lastterm: 6]
I161111 17:23:44.337447 264 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 70 [logterm: 0, index: 1051] rejected msgApp [logterm: 6, index: 1051] from 1
I161111 17:23:44.337613 105 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1044) from 70 for index 1051
I161111 17:23:44.337638 105 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 70 to [next = 1045, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.366990 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:112 > next_replica_id:113
I161111 17:23:44.371245 6907 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:112}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:44.373549 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 08b6259f at index 1053 in 50.813µs.
I161111 17:23:44.396992 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1284, log entries: 91
I161111 17:23:44.399137 452 storage/store.go:2988  [s3,r1/112:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:44.399241 452 storage/store.go:2988  [s3,r1/112:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:44.399804 6909 raft/raft.go:568  [s3,r1/112:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:44.399881 6909 raft/raft.go:316  [s3,r1/112:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 1053, applied: 1053, lastindex: 1053, lastterm: 6]
I161111 17:23:44.399959 6909 raft/raft.go:1106  [s3,r1/112:/M{in-ax}] ffffffffffffffff [commit: 1053] ignored snapshot [index: 1053, term: 6]
I161111 17:23:44.411242 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:113
I161111 17:23:44.417363 6732 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:113}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:113}]
I161111 17:23:44.608016 340 storage/store.go:2041  [s3,r1/112:/M{in-ax}] DEL 1
I161111 17:23:44.613150 273 raft/raft.go:568  [s3,r1/113:{-}] 71 became follower at term 0
I161111 17:23:44.617309 273 raft/raft.go:316  [s3,r1/113:{-}] newRaft 71 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:44.617348 273 raft/raft.go:568  [s3,r1/113:{-}] 71 became follower at term 1
I161111 17:23:44.617387 273 raft/raft.go:710  [s3,r1/113:{-}] 71 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:44.617415 273 raft/raft.go:568  [s3,r1/113:{-}] 71 became follower at term 6
I161111 17:23:44.640929 275 raft/raft.go:1088  [s3,r1/113:{-}] 71 [logterm: 0, index: 1058] rejected msgApp [logterm: 6, index: 1058] from 1
I161111 17:23:44.641339 74 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 71 for index 1058
I161111 17:23:44.641393 74 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 71 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.641852 74 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3b2e670d at index 1061 in 289.153µs.
I161111 17:23:44.642013 74 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 963, commit: 1061] sent snapshot[index: 1061, term: 6] to 71 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.642083 74 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 71 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1061]
I161111 17:23:44.795088 6862 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1293, log entries: 99
I161111 17:23:44.795428 6879 storage/store.go:1912  [s3] ADD 1
I161111 17:23:44.800958 6879 raft/raft.go:1125  [s3,r1/113:{-}] 71 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1061, term: 6]
I161111 17:23:44.800997 6879 raft/log.go:288  [s3,r1/113:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1061, term: 6]
I161111 17:23:44.801030 6879 raft/raft.go:1135  [s3,r1/113:{-}] 71 restored progress of 1 [next = 1062, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.801060 6879 raft/raft.go:1135  [s3,r1/113:{-}] 71 restored progress of 2 [next = 1062, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.801090 6879 raft/raft.go:1135  [s3,r1/113:{-}] 71 restored progress of 71 [next = 1062, match = 1061, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:44.801114 6879 raft/raft.go:1102  [s3,r1/113:{-}] 71 [commit: 1061] restored snapshot [index: 1061, term: 6]
I161111 17:23:44.801152 6879 storage/replica_raftstorage.go:589  [s3,r1/113:{-}] applying Raft snapshot at index 1061 (id=3b2e670d, encoded size=16, 1 rocksdb batches, 99 log entries)
I161111 17:23:44.939690 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 963-1067
I161111 17:23:45.074214 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:113 > next_replica_id:114
I161111 17:23:45.075904 7013 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:113}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:45.100242 6879 storage/replica_raftstorage.go:592  [s3,r1/113:/M{in-ax}] applied Raft snapshot in 0.272s
I161111 17:23:45.100741 452 storage/store.go:2988  [s3,r1/113:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:45.124393 6862 raft/raft.go:854  [s1,r1/1:/M{in-ax}] 1 no progress available for 71
I161111 17:23:45.124527 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7c226e89 at index 1073 in 66.614µs.
I161111 17:23:45.127509 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1305, log entries: 7
I161111 17:23:45.243537 340 storage/store.go:2041  [s3,r1/113:/M{in-ax}] DEL 1
I161111 17:23:45.244129 7048 storage/store.go:1912  [s3] ADD 1
I161111 17:23:45.244223 7048 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:45.247614 7048 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:45.247635 7048 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:45.247659 7048 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:45.247674 7048 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
W161111 17:23:45.268615 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 113}: raft group deleted
I161111 17:23:45.269075 7048 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1073, term: 6]
I161111 17:23:45.269115 7048 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1073, term: 6]
I161111 17:23:45.269153 7048 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1074, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.269183 7048 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1074, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.269207 7048 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1073] restored snapshot [index: 1073, term: 6]
I161111 17:23:45.269238 7048 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1073 (id=7c226e89, encoded size=16, 1 rocksdb batches, 7 log entries)
I161111 17:23:45.364242 7048 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.095s
I161111 17:23:45.364459 7048 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:45.365587 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:114
I161111 17:23:45.367138 7060 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:114}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:114}]
I161111 17:23:45.453729 286 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 72 became follower at term 6
I161111 17:23:45.453842 286 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 72 [peers: [1,2], term: 6, commit: 1073, applied: 1073, lastindex: 1073, lastterm: 6]
I161111 17:23:45.459426 240 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 72 [logterm: 0, index: 1078] rejected msgApp [logterm: 6, index: 1078] from 1
I161111 17:23:45.459591 103 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1073) from 72 for index 1078
I161111 17:23:45.459614 103 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 72 to [next = 1074, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.486063 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:114 > next_replica_id:115
I161111 17:23:45.487988 7079 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:114}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:45.490015 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9758e794 at index 1083 in 62.589µs.
I161111 17:23:45.494704 452 storage/store.go:2988  [s3,r1/114:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:45.511658 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1317, log entries: 17
I161111 17:23:45.586099 340 storage/store.go:2041  [s3,r1/114:/M{in-ax}] DEL 1
I161111 17:23:45.586668 7122 storage/store.go:1912  [s3] ADD 1
I161111 17:23:45.586755 7122 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:45.590113 7122 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:45.590135 7122 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:45.590157 7122 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:45.590170 7122 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
W161111 17:23:45.590447 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 114}: raft group deleted
I161111 17:23:45.593549 7122 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1083, term: 6]
I161111 17:23:45.593588 7122 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1083, term: 6]
I161111 17:23:45.593622 7122 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1084, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.593733 7122 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1084, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.593762 7122 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1083] restored snapshot [index: 1083, term: 6]
I161111 17:23:45.593809 7122 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1083 (id=9758e794, encoded size=16, 1 rocksdb batches, 17 log entries)
I161111 17:23:45.623468 7122 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.030s
I161111 17:23:45.623511 7122 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:45.623738 255 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 72 became follower at term 6
I161111 17:23:45.623869 255 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 72 [peers: [1,2], term: 6, commit: 1083, applied: 1083, lastindex: 1083, lastterm: 6]
I161111 17:23:45.624317 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:45.703614 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:115
I161111 17:23:45.705425 6954 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:115}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:115}]
I161111 17:23:45.896891 340 storage/store.go:2041  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:45.904265 259 raft/raft.go:568  [s3,r1/115:{-}] 73 became follower at term 0
I161111 17:23:45.907705 259 raft/raft.go:316  [s3,r1/115:{-}] newRaft 73 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:45.907724 259 raft/raft.go:568  [s3,r1/115:{-}] 73 became follower at term 1
I161111 17:23:45.907750 259 raft/raft.go:710  [s3,r1/115:{-}] 73 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:45.907761 259 raft/raft.go:568  [s3,r1/115:{-}] 73 became follower at term 6
I161111 17:23:45.938269 258 raft/raft.go:1088  [s3,r1/115:{-}] 73 [logterm: 0, index: 1088] rejected msgApp [logterm: 6, index: 1088] from 1
I161111 17:23:45.938420 104 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 73 for index 1088
I161111 17:23:45.938447 104 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 73 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.938586 104 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 065f982e at index 1091 in 87.871µs.
I161111 17:23:45.938625 104 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 1067, commit: 1091] sent snapshot[index: 1091, term: 6] to 73 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:45.938642 104 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 73 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1091]
I161111 17:23:46.010678 7099 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1326, log entries: 25
I161111 17:23:46.010775 7102 storage/store.go:1912  [s3] ADD 1
I161111 17:23:46.014264 7102 raft/raft.go:1125  [s3,r1/115:{-}] 73 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1091, term: 6]
I161111 17:23:46.014291 7102 raft/log.go:288  [s3,r1/115:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1091, term: 6]
I161111 17:23:46.014312 7102 raft/raft.go:1135  [s3,r1/115:{-}] 73 restored progress of 1 [next = 1092, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.014330 7102 raft/raft.go:1135  [s3,r1/115:{-}] 73 restored progress of 2 [next = 1092, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.014344 7102 raft/raft.go:1135  [s3,r1/115:{-}] 73 restored progress of 73 [next = 1092, match = 1091, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.014357 7102 raft/raft.go:1102  [s3,r1/115:{-}] 73 [commit: 1091] restored snapshot [index: 1091, term: 6]
I161111 17:23:46.014386 7102 storage/replica_raftstorage.go:589  [s3,r1/115:{-}] applying Raft snapshot at index 1091 (id=065f982e, encoded size=16, 1 rocksdb batches, 25 log entries)
I161111 17:23:46.076854 238 gossip/gossip.go:1071  [n3] node has connected to cluster via gossip
I161111 17:23:46.177603 7102 storage/replica_raftstorage.go:592  [s3,r1/115:/M{in-ax}] applied Raft snapshot in 0.163s
I161111 17:23:46.178142 74 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 73 [next = 1092, match = 1091, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1091]
I161111 17:23:46.264855 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:115 > next_replica_id:116
I161111 17:23:46.267293 7132 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:115}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:46.269634 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 4d337ab5 at index 1099 in 55.416µs.
I161111 17:23:46.277504 452 storage/store.go:2988  [s3,r1/115:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:46.285033 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1335, log entries: 33
I161111 17:23:46.342639 340 storage/store.go:2041  [s3,r1/115:/M{in-ax}] DEL 1
I161111 17:23:46.344175 7134 storage/store.go:1912  [s3] ADD 1
I161111 17:23:46.344655 7134 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:46.404338 7134 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:46.404504 7134 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:46.404665 7134 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:46.404736 7134 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:46.439643 7134 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1099, term: 6]
I161111 17:23:46.439735 7134 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1099, term: 6]
I161111 17:23:46.439830 7134 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1100, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.439904 7134 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1100, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.439987 7134 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1099] restored snapshot [index: 1099, term: 6]
I161111 17:23:46.440151 7134 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1099 (id=4d337ab5, encoded size=16, 1 rocksdb batches, 33 log entries)
I161111 17:23:46.633971 7134 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.194s
I161111 17:23:46.634024 7134 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:46.634789 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:116
I161111 17:23:46.641078 7230 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:116}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:116}]
I161111 17:23:46.733484 240 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 74 became follower at term 6
I161111 17:23:46.733569 240 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 74 [peers: [1,2], term: 6, commit: 1099, applied: 1099, lastindex: 1099, lastterm: 6]
I161111 17:23:46.733631 240 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 74 [logterm: 0, index: 1107] rejected msgApp [logterm: 6, index: 1107] from 1
I161111 17:23:46.735252 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1099) from 74 for index 1107
I161111 17:23:46.735288 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 74 to [next = 1100, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:46.799307 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:116 > next_replica_id:117
I161111 17:23:46.801191 7261 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:116}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:46.803189 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot d62b7784 at index 1112 in 53.224µs.
I161111 17:23:46.805111 452 storage/store.go:2988  [s3,r1/116:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:46.819243 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1350, log entries: 46
I161111 17:23:46.893841 340 storage/store.go:2041  [s3,r1/116:/M{in-ax}] DEL 1
W161111 17:23:47.032576 340 storage/store.go:785  storeMu: mutex held by github.com/cockroachdb/cockroach/pkg/storage.(*Store).removeReplicaImpl for 111.997349ms (>100ms):
goroutine 340 [running]:
runtime/debug.Stack(0x1c182d8, 0x1c18318, 0x47)
	/home/tamird/goroot/src/runtime/debug/stack.go:24 +0x79
github.com/cockroachdb/cockroach/pkg/util/syncutil.ThresholdLogger.func1(0x6acf1a5)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:65 +0xe1
github.com/cockroachdb/cockroach/pkg/util/syncutil.(*TimedMutex).Unlock(0xc4200942e0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/timedmutex.go:92 +0x80
github.com/cockroachdb/cockroach/pkg/storage.(*Store).removeReplicaImpl(0xc420094000, 0xc420b03400, 0x1, 0x277bef8, 0x0, 0x0, 0xc420422b48, 0x2, 0x8, 0xc420b66ac0, ...)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2044 +0xa2c
github.com/cockroachdb/cockroach/pkg/storage.(*Store).RemoveReplica(0xc420094000, 0xc420b03400, 0x1, 0x277bef8, 0x0, 0x0, 0xc420422b48, 0x2, 0x8, 0xc420b66ac0, ...)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1964 +0xea
github.com/cockroachdb/cockroach/pkg/storage.(*replicaGCQueue).process(0xc420011470, 0x7f530f25d7c0, 0xc4205f7c20, 0x7b, 0x4170, 0xc420b03400, 0xc4207b8800, 0x13, 0x20, 0x0, ...)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_gc_queue.go:189 +0x54c
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc42070e2d0, 0x7f530f21b878, 0xc420dd9260, 0xc420b03400, 0xc4201fc080, 0x0, 0x0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:541 +0x415
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2()
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:473 +0xa2
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc42065c480, 0xc420d03e58, 0x0, 0x0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:224 +0x100
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1()
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:477 +0x321
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42065c480, 0xc4201abe60)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66
I161111 17:23:47.052956 7288 storage/store.go:1912  [s3] ADD 1
I161111 17:23:47.053043 7288 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
W161111 17:23:47.056216 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 116}: raft group deleted
W161111 17:23:47.056245 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 116}: raft group deleted
W161111 17:23:47.056269 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 116}: raft group deleted
I161111 17:23:47.057014 7288 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:47.057051 7288 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:47.057087 7288 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:47.057111 7288 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:47.060312 7288 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1112, term: 6]
I161111 17:23:47.060356 7288 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1112, term: 6]
I161111 17:23:47.060391 7288 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.060425 7288 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.060449 7288 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1112] restored snapshot [index: 1112, term: 6]
I161111 17:23:47.060480 7288 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1112 (id=d62b7784, encoded size=16, 1 rocksdb batches, 46 log entries)
I161111 17:23:47.185671 7288 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.125s
I161111 17:23:47.185733 7288 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:47.186645 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:117
I161111 17:23:47.188227 7332 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:117}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:117}]
I161111 17:23:47.255338 252 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 75 became follower at term 6
I161111 17:23:47.255486 252 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 75 [peers: [1,2], term: 6, commit: 1112, applied: 1112, lastindex: 1112, lastterm: 6]
I161111 17:23:47.258932 259 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 75 [logterm: 0, index: 1120] rejected msgApp [logterm: 6, index: 1120] from 1
I161111 17:23:47.259087 81 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1112) from 75 for index 1120
I161111 17:23:47.259112 81 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 75 to [next = 1113, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.284368 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:117 > next_replica_id:118
I161111 17:23:47.321284 7294 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:117}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:47.322945 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 9dfd973e at index 1125 in 55.356µs.
I161111 17:23:47.399767 7328 raft/raft.go:568  [s3,r1/117:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:47.399816 7328 raft/raft.go:316  [s3,r1/117:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,75], term: 6, commit: 1120, applied: 1120, lastindex: 1121, lastterm: 6]
I161111 17:23:47.399881 7328 raft/raft.go:1125  [s3,r1/117:/M{in-ax}] ffffffffffffffff [commit: 1120, lastindex: 1121, lastterm: 6] starts to restore snapshot [index: 1125, term: 6]
I161111 17:23:47.399901 7328 raft/log.go:288  [s3,r1/117:/M{in-ax}] log [committed=1120, applied=1120, unstable.offset=1122, len(unstable.Entries)=0] starts to restore snapshot [index: 1125, term: 6]
I161111 17:23:47.399923 7328 raft/raft.go:1135  [s3,r1/117:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 1126, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.399939 7328 raft/raft.go:1135  [s3,r1/117:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 1126, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.399952 7328 raft/raft.go:1102  [s3,r1/117:/M{in-ax}] ffffffffffffffff [commit: 1125] restored snapshot [index: 1125, term: 6]
I161111 17:23:47.399971 7328 storage/replica_raftstorage.go:589  [s3,r1/117:/M{in-ax}] applying Raft snapshot at index 1125 (id=9dfd973e, encoded size=16, 1 rocksdb batches, 59 log entries)
I161111 17:23:47.412325 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1365, log entries: 59
I161111 17:23:47.579299 7328 storage/replica_raftstorage.go:592  [s3,r1/117:/M{in-ax}] applied Raft snapshot in 0.179s
I161111 17:23:47.579837 261 raft/raft.go:568  [s3,r1/117:/M{in-ax}] 75 became follower at term 6
I161111 17:23:47.579977 261 raft/raft.go:316  [s3,r1/117:/M{in-ax}] newRaft 75 [peers: [1,2], term: 6, commit: 1125, applied: 1125, lastindex: 1125, lastterm: 6]
I161111 17:23:47.588663 452 storage/store.go:2988  [s3,r1/117:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:47.588868 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:118
I161111 17:23:47.589859 452 storage/store.go:2988  [s3,r1/117:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:47.590848 7353 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:118}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:118}]
I161111 17:23:47.673079 340 storage/store.go:2041  [s3,r1/117:/M{in-ax}] DEL 1
I161111 17:23:47.754235 270 raft/raft.go:568  [s3,r1/118:{-}] 76 became follower at term 0
I161111 17:23:47.754484 270 raft/raft.go:316  [s3,r1/118:{-}] newRaft 76 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:47.754513 270 raft/raft.go:568  [s3,r1/118:{-}] 76 became follower at term 1
I161111 17:23:47.754538 270 raft/raft.go:710  [s3,r1/118:{-}] 76 [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:47.754551 270 raft/raft.go:568  [s3,r1/118:{-}] 76 became follower at term 6
I161111 17:23:47.791003 272 raft/raft.go:1088  [s3,r1/118:{-}] 76 [logterm: 0, index: 1130] rejected msgApp [logterm: 6, index: 1130] from 1
I161111 17:23:47.791183 87 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 76 for index 1130
I161111 17:23:47.791219 87 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 76 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.791379 87 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3e2399b5 at index 1133 in 103.704µs.
I161111 17:23:47.791424 87 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 1067, commit: 1133] sent snapshot[index: 1133, term: 6] to 76 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.791442 87 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 76 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1133]
I161111 17:23:47.794525 7403 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1374, log entries: 67
I161111 17:23:47.794610 7406 storage/store.go:1912  [s3] ADD 1
I161111 17:23:47.794870 7406 raft/raft.go:1125  [s3,r1/118:{-}] 76 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1133, term: 6]
I161111 17:23:47.795065 7406 raft/log.go:288  [s3,r1/118:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1133, term: 6]
I161111 17:23:47.795101 7406 raft/raft.go:1135  [s3,r1/118:{-}] 76 restored progress of 1 [next = 1134, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.795156 7406 raft/raft.go:1135  [s3,r1/118:{-}] 76 restored progress of 2 [next = 1134, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.795187 7406 raft/raft.go:1135  [s3,r1/118:{-}] 76 restored progress of 76 [next = 1134, match = 1133, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.795234 7406 raft/raft.go:1102  [s3,r1/118:{-}] 76 [commit: 1133] restored snapshot [index: 1133, term: 6]
I161111 17:23:47.795272 7406 storage/replica_raftstorage.go:589  [s3,r1/118:{-}] applying Raft snapshot at index 1133 (id=3e2399b5, encoded size=16, 1 rocksdb batches, 67 log entries)
I161111 17:23:47.868087 7406 storage/replica_raftstorage.go:592  [s3,r1/118:/M{in-ax}] applied Raft snapshot in 0.073s
I161111 17:23:47.868517 111 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 76 [next = 1134, match = 1133, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1133]
I161111 17:23:47.893725 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:118 > next_replica_id:119
I161111 17:23:47.895692 7417 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:118}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:47.906846 452 storage/store.go:2988  [s3,r1/118:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:47.908915 452 storage/store.go:2988  [s3,r1/118:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:47.914342 340 storage/store.go:2041  [s3,r1/118:/M{in-ax}] DEL 1
I161111 17:23:47.942006 7403 raft/raft.go:854  [s1,r1/1:/M{in-ax}] 1 no progress available for 76
I161111 17:23:47.942190 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1e444bc4 at index 1138 in 89.9µs.
I161111 17:23:47.945657 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1380, log entries: 72
I161111 17:23:47.946450 7428 storage/store.go:1912  [s3] ADD 1
I161111 17:23:47.946591 7428 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:47.947183 7428 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:47.947243 7428 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:47.947281 7428 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:47.947327 7428 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:47.947746 7428 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1138, term: 6]
I161111 17:23:47.947781 7428 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1138, term: 6]
I161111 17:23:47.947833 7428 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1139, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.947862 7428 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1139, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:47.947904 7428 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1138] restored snapshot [index: 1138, term: 6]
I161111 17:23:47.947935 7428 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1138 (id=1e444bc4, encoded size=16, 1 rocksdb batches, 72 log entries)
I161111 17:23:48.052056 7428 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.104s
I161111 17:23:48.052145 7428 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:48.052999 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:119
I161111 17:23:48.054594 7460 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:119}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:119}]
I161111 17:23:48.153731 286 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 77 became follower at term 6
I161111 17:23:48.153777 286 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 77 [peers: [1,2], term: 6, commit: 1138, applied: 1138, lastindex: 1138, lastterm: 6]
I161111 17:23:48.160512 240 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 77 [logterm: 0, index: 1143] rejected msgApp [logterm: 6, index: 1143] from 1
I161111 17:23:48.161555 74 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1138) from 77 for index 1143
I161111 17:23:48.161579 74 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 77 to [next = 1139, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.209094 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:119 > next_replica_id:120
I161111 17:23:48.217671 7206 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:119}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:48.240148 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 0c065d67 at index 1148 in 55.484µs.
I161111 17:23:48.242400 452 storage/store.go:2988  [s3,r1/119:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:48.271062 340 storage/store.go:2041  [s3,r1/119:/M{in-ax}] DEL 1
I161111 17:23:48.340353 7376 storage/store.go:1912  [s3] ADD 1
I161111 17:23:48.340442 7376 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:48.340991 7376 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:48.341010 7376 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:48.341033 7376 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:48.341047 7376 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:48.341425 7376 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1148, term: 6]
I161111 17:23:48.341448 7376 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1148, term: 6]
I161111 17:23:48.341472 7376 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1149, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.341498 7376 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1149, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.341513 7376 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1148] restored snapshot [index: 1148, term: 6]
I161111 17:23:48.341530 7376 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1148 (id=0c065d67, encoded size=16, 1 rocksdb batches, 82 log entries)
I161111 17:23:48.344867 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1392, log entries: 82
I161111 17:23:48.372077 7376 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.031s
I161111 17:23:48.372161 7376 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:48.372474 264 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 77 became follower at term 6
I161111 17:23:48.372540 264 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 77 [peers: [1,2], term: 6, commit: 1148, applied: 1148, lastindex: 1148, lastterm: 6]
I161111 17:23:48.373196 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:120
I161111 17:23:48.374841 7556 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:120}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:120}]
I161111 17:23:48.387769 452 storage/store.go:2988  [s3,r1/?:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:48.453570 266 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 78 became follower at term 6
I161111 17:23:48.453618 266 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 78 [peers: [1,2], term: 6, commit: 1148, applied: 1148, lastindex: 1148, lastterm: 6]
I161111 17:23:48.487571 270 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 78 [logterm: 0, index: 1153] rejected msgApp [logterm: 6, index: 1153] from 1
I161111 17:23:48.487731 85 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1148) from 78 for index 1153
I161111 17:23:48.487762 85 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 78 to [next = 1149, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.517527 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:120 > next_replica_id:121
I161111 17:23:48.519642 7560 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:120}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:48.528611 452 storage/store.go:2988  [s3,r1/120:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:48.540482 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 19d79447 at index 1156 in 60.341µs.
I161111 17:23:48.547663 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1402, log entries: 90
I161111 17:23:48.638446 340 storage/store.go:2041  [s3,r1/120:/M{in-ax}] DEL 1
I161111 17:23:48.639001 7577 storage/store.go:1912  [s3] ADD 1
I161111 17:23:48.639123 7577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:48.639789 7577 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:48.639829 7577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:48.639866 7577 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:48.639910 7577 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:48.640503 7577 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1156, term: 6]
I161111 17:23:48.640549 7577 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1156, term: 6]
I161111 17:23:48.640605 7577 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.640640 7577 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.640661 7577 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1156] restored snapshot [index: 1156, term: 6]
I161111 17:23:48.640711 7577 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1156 (id=19d79447, encoded size=16, 1 rocksdb batches, 90 log entries)
I161111 17:23:48.669080 7577 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.028s
I161111 17:23:48.669122 7577 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:48.670122 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:121
I161111 17:23:48.671822 7582 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:121}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:121}]
I161111 17:23:48.698431 284 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 79 became follower at term 6
I161111 17:23:48.698550 284 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 79 [peers: [1,2], term: 6, commit: 1156, applied: 1156, lastindex: 1156, lastterm: 6]
I161111 17:23:48.698569 284 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 79 [logterm: 0, index: 1160] rejected msgApp [logterm: 6, index: 1160] from 1
I161111 17:23:48.698729 85 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1156) from 79 for index 1160
I161111 17:23:48.698752 85 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 79 to [next = 1157, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.732000 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:121 > next_replica_id:122
I161111 17:23:48.743976 7613 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:121}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:48.780683 452 storage/store.go:2988  [s3,r1/121:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:48.788278 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 94c3d05a at index 1165 in 64.797µs.
I161111 17:23:48.791722 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1413, log entries: 99
I161111 17:23:48.792196 452 storage/store.go:2988  [s3,r1/121:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:48.814853 340 storage/store.go:2041  [s3,r1/121:/M{in-ax}] DEL 1
I161111 17:23:48.815381 7516 storage/store.go:1912  [s3] ADD 1
I161111 17:23:48.815516 7516 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:48.817614 7516 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:48.817663 7516 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:48.817697 7516 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:48.817736 7516 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:48.826231 7516 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1165, term: 6]
I161111 17:23:48.826279 7516 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1165, term: 6]
I161111 17:23:48.826313 7516 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1166, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.826340 7516 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1166, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:48.826365 7516 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1165] restored snapshot [index: 1165, term: 6]
I161111 17:23:48.826397 7516 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1165 (id=94c3d05a, encoded size=16, 1 rocksdb batches, 99 log entries)
I161111 17:23:48.948507 7516 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.122s
I161111 17:23:48.948837 7516 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:48.950259 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:122
I161111 17:23:48.951703 7637 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:122}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:122}]
I161111 17:23:49.053671 252 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7a became follower at term 6
I161111 17:23:49.053716 252 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7a [peers: [1,2], term: 6, commit: 1165, applied: 1165, lastindex: 1165, lastterm: 6]
I161111 17:23:49.058472 259 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7a [logterm: 0, index: 1170] rejected msgApp [logterm: 6, index: 1170] from 1
I161111 17:23:49.058631 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1165) from 7a for index 1170
I161111 17:23:49.058659 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7a to [next = 1166, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.059586 152 storage/raft_log_queue.go:219  [raftlog,s1,r1/1:/M{in-ax}] truncating raft log 1067-1171
I161111 17:23:49.112352 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:122 > next_replica_id:123
I161111 17:23:49.124288 7489 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:122}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:49.126164 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot dcdbfa66 at index 1176 in 53.728µs.
I161111 17:23:49.128064 452 storage/store.go:2988  [s3,r1/122:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:49.141111 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1425, log entries: 6
I161111 17:23:49.174308 340 storage/store.go:2041  [s3,r1/122:/M{in-ax}] DEL 1
I161111 17:23:49.179964 7695 storage/store.go:1912  [s3] ADD 1
I161111 17:23:49.180048 7695 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:49.181932 7695 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:49.181959 7695 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:49.181982 7695 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:49.181994 7695 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:49.183546 7695 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1176, term: 6]
I161111 17:23:49.183567 7695 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1176, term: 6]
I161111 17:23:49.183589 7695 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1177, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.183606 7695 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1177, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.183619 7695 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1176] restored snapshot [index: 1176, term: 6]
I161111 17:23:49.183638 7695 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1176 (id=dcdbfa66, encoded size=16, 1 rocksdb batches, 6 log entries)
W161111 17:23:49.184209 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 122}: raft group deleted
W161111 17:23:49.184235 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 122}: raft group deleted
W161111 17:23:49.184255 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 122}: raft group deleted
I161111 17:23:49.198355 7695 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.015s
I161111 17:23:49.200934 7695 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:49.201749 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:123
I161111 17:23:49.203370 7617 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:123}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:123}]
I161111 17:23:49.241251 271 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7b became follower at term 6
I161111 17:23:49.241378 271 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7b [peers: [1,2], term: 6, commit: 1176, applied: 1176, lastindex: 1176, lastterm: 6]
I161111 17:23:49.241400 271 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7b [logterm: 0, index: 1178] rejected msgApp [logterm: 6, index: 1178] from 1
I161111 17:23:49.241568 91 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1176) from 7b for index 1178
I161111 17:23:49.241591 91 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7b to [next = 1177, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.252196 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:123 > next_replica_id:124
I161111 17:23:49.253797 7706 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:123}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:49.255672 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 3e6cd6df at index 1181 in 56.833µs.
I161111 17:23:49.271291 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1432, log entries: 11
I161111 17:23:49.271493 7243 raft/raft.go:568  [s3,r1/123:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:49.271528 7243 raft/raft.go:316  [s3,r1/123:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2,7b], term: 6, commit: 1178, applied: 1178, lastindex: 1179, lastterm: 6]
I161111 17:23:49.271589 7243 raft/raft.go:1125  [s3,r1/123:/M{in-ax}] ffffffffffffffff [commit: 1178, lastindex: 1179, lastterm: 6] starts to restore snapshot [index: 1181, term: 6]
I161111 17:23:49.271610 7243 raft/log.go:288  [s3,r1/123:/M{in-ax}] log [committed=1178, applied=1178, unstable.offset=1180, len(unstable.Entries)=0] starts to restore snapshot [index: 1181, term: 6]
I161111 17:23:49.271631 7243 raft/raft.go:1135  [s3,r1/123:/M{in-ax}] ffffffffffffffff restored progress of 1 [next = 1182, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.271646 7243 raft/raft.go:1135  [s3,r1/123:/M{in-ax}] ffffffffffffffff restored progress of 2 [next = 1182, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.271659 7243 raft/raft.go:1102  [s3,r1/123:/M{in-ax}] ffffffffffffffff [commit: 1181] restored snapshot [index: 1181, term: 6]
I161111 17:23:49.271677 7243 storage/replica_raftstorage.go:589  [s3,r1/123:/M{in-ax}] applying Raft snapshot at index 1181 (id=3e6cd6df, encoded size=16, 1 rocksdb batches, 11 log entries)
I161111 17:23:49.276434 452 storage/store.go:2988  [s3,r1/123:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:49.384255 7243 storage/replica_raftstorage.go:592  [s3,r1/123:/M{in-ax}] applied Raft snapshot in 0.113s
I161111 17:23:49.408101 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:124
I161111 17:23:49.409663 7520 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:124}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:124}]
I161111 17:23:49.463830 340 storage/store.go:2041  [s3,r1/123:/M{in-ax}] DEL 1
I161111 17:23:49.518278 269 raft/raft.go:568  [s3,r1/124:{-}] 7c became follower at term 0
I161111 17:23:49.520236 269 raft/raft.go:316  [s3,r1/124:{-}] newRaft 7c [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:49.520256 269 raft/raft.go:568  [s3,r1/124:{-}] 7c became follower at term 1
I161111 17:23:49.520278 269 raft/raft.go:710  [s3,r1/124:{-}] 7c [term: 1] received a MsgHeartbeat message with higher term from 1 [term: 6]
I161111 17:23:49.520289 269 raft/raft.go:568  [s3,r1/124:{-}] 7c became follower at term 6
I161111 17:23:49.544966 278 raft/raft.go:1088  [s3,r1/124:{-}] 7c [logterm: 0, index: 1185] rejected msgApp [logterm: 6, index: 1185] from 1
I161111 17:23:49.545331 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 7c for index 1185
I161111 17:23:49.545449 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7c to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.545689 108 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 1f8e0e71 at index 1188 in 160.084µs.
I161111 17:23:49.545820 108 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 1171, commit: 1188] sent snapshot[index: 1188, term: 6] to 7c [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.545857 108 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 7c [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1188]
I161111 17:23:49.553482 7678 storage/store.go:1912  [s3] ADD 1
I161111 17:23:49.555808 7678 raft/raft.go:1125  [s3,r1/124:{-}] 7c [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1188, term: 6]
I161111 17:23:49.555911 7678 raft/log.go:288  [s3,r1/124:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1188, term: 6]
I161111 17:23:49.555943 7678 raft/raft.go:1135  [s3,r1/124:{-}] 7c restored progress of 1 [next = 1189, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.555966 7678 raft/raft.go:1135  [s3,r1/124:{-}] 7c restored progress of 2 [next = 1189, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.555989 7678 raft/raft.go:1135  [s3,r1/124:{-}] 7c restored progress of 7c [next = 1189, match = 1188, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.556011 7678 raft/raft.go:1102  [s3,r1/124:{-}] 7c [commit: 1188] restored snapshot [index: 1188, term: 6]
I161111 17:23:49.556103 7678 storage/replica_raftstorage.go:589  [s3,r1/124:{-}] applying Raft snapshot at index 1188 (id=1f8e0e71, encoded size=16, 1 rocksdb batches, 18 log entries)
I161111 17:23:49.569015 7676 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1440, log entries: 18
I161111 17:23:49.686782 7678 storage/replica_raftstorage.go:592  [s3,r1/124:/M{in-ax}] applied Raft snapshot in 0.131s
I161111 17:23:49.688371 73 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 7c [next = 1189, match = 1188, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1188]
I161111 17:23:49.754990 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:124 > next_replica_id:125
I161111 17:23:49.757045 7767 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:124}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:49.759615 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot a8667e11 at index 1193 in 53.647µs.
I161111 17:23:49.760640 452 storage/store.go:2988  [s3,r1/124:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:49.763946 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1446, log entries: 23
I161111 17:23:49.785232 340 storage/store.go:2041  [s3,r1/124:/M{in-ax}] DEL 1
I161111 17:23:49.856889 7781 storage/store.go:1912  [s3] ADD 1
I161111 17:23:49.856976 7781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
W161111 17:23:49.858702 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 124}: raft group deleted
I161111 17:23:49.858966 7781 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:49.858986 7781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:49.859029 7781 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:49.859046 7781 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:49.861878 7781 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1193, term: 6]
I161111 17:23:49.861913 7781 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1193, term: 6]
I161111 17:23:49.861959 7781 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.861985 7781 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:49.862008 7781 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1193] restored snapshot [index: 1193, term: 6]
I161111 17:23:49.862051 7781 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1193 (id=a8667e11, encoded size=16, 1 rocksdb batches, 23 log entries)
I161111 17:23:49.910422 7781 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.048s
I161111 17:23:49.916745 7781 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:49.917545 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:125
I161111 17:23:49.919966 7800 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:125}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:125}]
I161111 17:23:50.022044 241 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7d became follower at term 6
I161111 17:23:50.022110 241 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7d [peers: [1,2], term: 6, commit: 1193, applied: 1193, lastindex: 1193, lastterm: 6]
I161111 17:23:50.022137 241 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7d [logterm: 0, index: 1198] rejected msgApp [logterm: 6, index: 1198] from 1
I161111 17:23:50.022533 107 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1193) from 7d for index 1198
I161111 17:23:50.022575 107 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7d to [next = 1194, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.028328 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:125 > next_replica_id:126
I161111 17:23:50.040462 7807 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:125}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:50.042811 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 08ee6fc9 at index 1203 in 55.048µs.
I161111 17:23:50.049112 452 storage/store.go:2988  [s3,r1/125:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:50.053749 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1458, log entries: 33
I161111 17:23:50.101021 340 storage/store.go:2041  [s3,r1/125:/M{in-ax}] DEL 1
I161111 17:23:50.101745 7833 storage/store.go:1912  [s3] ADD 1
I161111 17:23:50.101828 7833 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:50.103878 7833 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:50.103901 7833 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:50.103926 7833 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:50.103940 7833 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:50.108983 7833 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1203, term: 6]
I161111 17:23:50.109051 7833 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1203, term: 6]
I161111 17:23:50.109086 7833 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1204, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.109137 7833 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1204, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.109159 7833 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1203] restored snapshot [index: 1203, term: 6]
I161111 17:23:50.109217 7833 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1203 (id=08ee6fc9, encoded size=16, 1 rocksdb batches, 33 log entries)
I161111 17:23:50.286893 7833 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.178s
I161111 17:23:50.287025 7833 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:50.287366 256 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7d became follower at term 6
I161111 17:23:50.287403 256 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7d [peers: [1,2], term: 6, commit: 1203, applied: 1203, lastindex: 1203, lastterm: 6]
I161111 17:23:50.288391 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:126
I161111 17:23:50.289907 7494 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:126}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:126}]
I161111 17:23:50.320654 258 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7e became follower at term 6
I161111 17:23:50.320701 258 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7e [peers: [1,2], term: 6, commit: 1203, applied: 1203, lastindex: 1203, lastterm: 6]
I161111 17:23:50.320747 258 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7e [logterm: 0, index: 1208] rejected msgApp [logterm: 6, index: 1208] from 1
I161111 17:23:50.320910 70 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1203) from 7e for index 1208
I161111 17:23:50.320936 70 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7e to [next = 1204, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.344703 452 storage/store.go:2988  [s3,r1/126:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:50.346248 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:126 > next_replica_id:127
I161111 17:23:50.350139 7661 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:126}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:50.353440 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot ce0f355f at index 1212 in 52.788µs.
I161111 17:23:50.356679 452 storage/store.go:2988  [s3,r1/126:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:50.458378 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1469, log entries: 42
I161111 17:23:50.471936 340 storage/store.go:2041  [s3,r1/126:/M{in-ax}] DEL 1
W161111 17:23:50.472399 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 126}: raft group deleted
W161111 17:23:50.472468 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 126}: raft group deleted
I161111 17:23:50.472672 7908 storage/store.go:1912  [s3] ADD 1
I161111 17:23:50.472761 7908 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:50.475051 7908 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:50.475071 7908 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:50.475094 7908 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:50.475107 7908 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:50.476977 7908 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1212, term: 6]
I161111 17:23:50.477010 7908 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1212, term: 6]
I161111 17:23:50.477041 7908 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1213, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.477061 7908 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1213, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.477082 7908 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1212] restored snapshot [index: 1212, term: 6]
I161111 17:23:50.477274 7908 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1212 (id=ce0f355f, encoded size=16, 1 rocksdb batches, 42 log entries)
I161111 17:23:50.595364 7908 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.118s
I161111 17:23:50.595452 7908 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:50.596600 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:127
I161111 17:23:50.598114 7756 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:127}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:127}]
I161111 17:23:50.653738 273 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 7f became follower at term 6
I161111 17:23:50.653786 273 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 7f [peers: [1,2], term: 6, commit: 1212, applied: 1212, lastindex: 1212, lastterm: 6]
I161111 17:23:50.678451 275 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 7f [logterm: 0, index: 1218] rejected msgApp [logterm: 6, index: 1218] from 1
I161111 17:23:50.678625 85 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1212) from 7f for index 1218
I161111 17:23:50.678649 85 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 7f to [next = 1213, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:50.739288 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:127 > next_replica_id:128
I161111 17:23:50.741196 7851 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:127}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:50.760642 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 44cbdfdb at index 1223 in 61.474µs.
I161111 17:23:50.763868 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1482, log entries: 53
I161111 17:23:50.764037 7964 raft/raft.go:568  [s3,r1/127:/M{in-ax}] ffffffffffffffff became follower at term 6
I161111 17:23:50.764073 7964 raft/raft.go:316  [s3,r1/127:/M{in-ax}] newRaft ffffffffffffffff [peers: [1,2], term: 6, commit: 1223, applied: 1223, lastindex: 1223, lastterm: 6]
I161111 17:23:50.769943 7964 raft/raft.go:1106  [s3,r1/127:/M{in-ax}] ffffffffffffffff [commit: 1223] ignored snapshot [index: 1223, term: 6]
I161111 17:23:50.770306 452 storage/store.go:2988  [s3,r1/127:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:50.775016 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:128
I161111 17:23:50.776643 7856 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:128}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:128}]
I161111 17:23:50.787301 452 storage/store.go:2988  [s3,r1/127:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:50.916523 340 storage/store.go:2041  [s3,r1/127:/M{in-ax}] DEL 1
I161111 17:23:51.040137 247 raft/raft.go:568  [s3,r1/128:{-}] 80 became follower at term 0
I161111 17:23:51.042287 247 raft/raft.go:316  [s3,r1/128:{-}] newRaft 80 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:51.042306 247 raft/raft.go:568  [s3,r1/128:{-}] 80 became follower at term 1
I161111 17:23:51.042329 247 raft/raft.go:710  [s3,r1/128:{-}] 80 [term: 1] received a MsgApp message with higher term from 1 [term: 6]
I161111 17:23:51.042343 247 raft/raft.go:568  [s3,r1/128:{-}] 80 became follower at term 6
I161111 17:23:51.042361 247 raft/raft.go:1088  [s3,r1/128:{-}] 80 [logterm: 0, index: 1225] rejected msgApp [logterm: 6, index: 1225] from 1
I161111 17:23:51.042607 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 0) from 80 for index 1225
I161111 17:23:51.042632 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 80 to [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.042754 108 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 43f8b575 at index 1228 in 75.629µs.
I161111 17:23:51.042799 108 raft/raft.go:400  [s1,r1/1:/M{in-ax}] 1 [firstindex: 1171, commit: 1228] sent snapshot[index: 1228, term: 6] to 80 [next = 1, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.042817 108 raft/raft.go:402  [s1,r1/1:/M{in-ax}] 1 paused sending replication messages to 80 [next = 1, match = 0, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1228]
I161111 17:23:51.047465 7993 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1488, log entries: 58
I161111 17:23:51.047555 7977 storage/store.go:1912  [s3] ADD 1
I161111 17:23:51.063413 7977 raft/raft.go:1125  [s3,r1/128:{-}] 80 [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1228, term: 6]
I161111 17:23:51.063458 7977 raft/log.go:288  [s3,r1/128:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1228, term: 6]
I161111 17:23:51.063601 7977 raft/raft.go:1135  [s3,r1/128:{-}] 80 restored progress of 1 [next = 1229, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.063633 7977 raft/raft.go:1135  [s3,r1/128:{-}] 80 restored progress of 2 [next = 1229, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.063779 7977 raft/raft.go:1135  [s3,r1/128:{-}] 80 restored progress of 80 [next = 1229, match = 1228, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.063990 7977 raft/raft.go:1102  [s3,r1/128:{-}] 80 [commit: 1228] restored snapshot [index: 1228, term: 6]
I161111 17:23:51.064045 7977 storage/replica_raftstorage.go:589  [s3,r1/128:{-}] applying Raft snapshot at index 1228 (id=43f8b575, encoded size=16, 1 rocksdb batches, 58 log entries)
I161111 17:23:51.194345 7977 storage/replica_raftstorage.go:592  [s3,r1/128:/M{in-ax}] applied Raft snapshot in 0.130s
I161111 17:23:51.194822 75 raft/raft.go:878  [s1,r1/1:/M{in-ax}] 1 snapshot aborted, resumed sending replication messages to 80 [next = 1229, match = 1228, state = ProgressStateSnapshot, waiting = true, pendingSnapshot = 1228]
I161111 17:23:51.334248 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:128 > next_replica_id:129
I161111 17:23:51.336104 8020 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:128}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:51.338598 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot bfee3de0 at index 1236 in 51.379µs.
I161111 17:23:51.339264 452 storage/store.go:2988  [s3,r1/128:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:51.344960 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1497, log entries: 66
I161111 17:23:51.458680 340 storage/store.go:2041  [s3,r1/128:/M{in-ax}] DEL 1
I161111 17:23:51.459373 8066 storage/store.go:1912  [s3] ADD 1
I161111 17:23:51.459462 8066 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:51.461724 8066 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:51.461745 8066 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:51.461768 8066 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:51.461780 8066 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
W161111 17:23:51.464267 436 storage/store.go:2992  [s1] got error from range 1, replica {3 3 128}: raft group deleted
I161111 17:23:51.467201 8066 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1236, term: 6]
I161111 17:23:51.467237 8066 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1236, term: 6]
I161111 17:23:51.467272 8066 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.467298 8066 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.467320 8066 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1236] restored snapshot [index: 1236, term: 6]
I161111 17:23:51.467350 8066 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1236 (id=bfee3de0, encoded size=16, 1 rocksdb batches, 66 log entries)
I161111 17:23:51.550500 8066 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.083s
I161111 17:23:51.550640 8066 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:51.551529 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > next_replica_id:129
I161111 17:23:51.553112 8100 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing ADD_REPLICA {NodeID:3 StoreID:3 ReplicaID:129}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2} {NodeID:3 StoreID:3 ReplicaID:129}]
I161111 17:23:51.579535 275 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 81 became follower at term 6
I161111 17:23:51.579582 275 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 81 [peers: [1,2], term: 6, commit: 1236, applied: 1236, lastindex: 1236, lastterm: 6]
I161111 17:23:51.579629 275 raft/raft.go:1088  [s3,r1/?:/M{in-ax}] 81 [logterm: 0, index: 1241] rejected msgApp [logterm: 6, index: 1241] from 1
I161111 17:23:51.580660 108 raft/raft.go:863  [s1,r1/1:/M{in-ax}] 1 received msgApp rejection(lastindex: 1236) from 81 for index 1241
I161111 17:23:51.580686 108 raft/raft.go:865  [s1,r1/1:/M{in-ax}] 1 decreased progress of 81 to [next = 1237, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.584754 25 storage/replica_command.go:3245  [s1,r1/1:/M{in-ax}] change replicas: read existing descriptor range_id:1 start_key:"" end_key:"\377\377" replicas:<node_id:1 store_id:1 replica_id:1 > replicas:<node_id:2 store_id:2 replica_id:2 > replicas:<node_id:3 store_id:3 replica_id:129 > next_replica_id:130
I161111 17:23:51.594005 8112 storage/replica.go:2052  [s1,r1/1:/M{in-ax}] proposing REMOVE_REPLICA {NodeID:3 StoreID:3 ReplicaID:129}: [{NodeID:1 StoreID:1 ReplicaID:1} {NodeID:2 StoreID:2 ReplicaID:2}]
I161111 17:23:51.611058 25 storage/replica_raftstorage.go:445  [s1,r1/1:/M{in-ax}] generated snapshot 7f558669 at index 1244 in 60.767µs.
I161111 17:23:51.640968 452 storage/store.go:2988  [s3,r1/129:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:51.658568 25 storage/store.go:3131  [s1,r1/1:/M{in-ax}] streamed snapshot: kv pairs: 1507, log entries: 74
I161111 17:23:51.659319 452 storage/store.go:2988  [s3,r1/129:/M{in-ax}] added to replica GC queue (peer suggestion)
I161111 17:23:51.820728 340 storage/store.go:2041  [s3,r1/129:/M{in-ax}] DEL 1
I161111 17:23:51.821323 8130 storage/store.go:1912  [s3] ADD 1
I161111 17:23:51.821407 8130 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 0
I161111 17:23:51.824230 8130 raft/raft.go:316  [s3,r1/?:{-}] newRaft ffffffffffffffff [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
I161111 17:23:51.824263 8130 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 1
I161111 17:23:51.824319 8130 raft/raft.go:710  [s3,r1/?:{-}] ffffffffffffffff [term: 1] received a MsgSnap message with higher term from 1 [term: 6]
I161111 17:23:51.824345 8130 raft/raft.go:568  [s3,r1/?:{-}] ffffffffffffffff became follower at term 6
I161111 17:23:51.830431 8130 raft/raft.go:1125  [s3,r1/?:{-}] ffffffffffffffff [commit: 0, lastindex: 0, lastterm: 0] starts to restore snapshot [index: 1244, term: 6]
I161111 17:23:51.830476 8130 raft/log.go:288  [s3,r1/?:{-}] log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 1244, term: 6]
I161111 17:23:51.830524 8130 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 1 [next = 1245, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.830592 8130 raft/raft.go:1135  [s3,r1/?:{-}] ffffffffffffffff restored progress of 2 [next = 1245, match = 0, state = ProgressStateProbe, waiting = false, pendingSnapshot = 0]
I161111 17:23:51.830616 8130 raft/raft.go:1102  [s3,r1/?:{-}] ffffffffffffffff [commit: 1244] restored snapshot [index: 1244, term: 6]
I161111 17:23:51.830644 8130 storage/replica_raftstorage.go:589  [s3,r1/?:{-}] applying preemptive snapshot at index 1244 (id=7f558669, encoded size=16, 1 rocksdb batches, 74 log entries)
I161111 17:23:52.025617 8130 storage/replica_raftstorage.go:592  [s3,r1/?:/M{in-ax}] applied preemptive snapshot in 0.195s
I161111 17:23:52.025958 8130 storage/store.go:2817  [s3,r1/?:/M{in-ax}] DEL 1
I161111 17:23:52.026161 249 raft/raft.go:568  [s3,r1/?:/M{in-ax}] 81 became follower at term 6
I161111 17:23:52.026205 249 raft/raft.go:316  [s3,r1/?:/M{in-ax}] newRaft 81 [peers: [1,2], term: 6, commit: 1244, applied: 1244, lastindex: 1244, lastterm: 6]
E161111 17:23:52.026265 249 raft/log.go:191  [s3,r1/?:/M{in-ax}] tocommit(1246) is out of range [lastIndex(1244)]. Was the raft log corrupted, truncated, or lost?
panic: tocommit(1246) is out of range [lastIndex(1244)]. Was the raft log corrupted, truncated, or lost? [recovered]
	panic: tocommit(1246) is out of range [lastIndex(1244)]. Was the raft log corrupted, truncated, or lost?

goroutine 249 [running]:
panic(0x16fb740, 0xc42072ebd0)
	/home/tamird/goroot/src/runtime/panic.go:500 +0x1a1
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc42065c480)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:185 +0x6e
panic(0x16fb740, 0xc42072ebd0)
	/home/tamird/goroot/src/runtime/panic.go:458 +0x243
github.com/cockroachdb/cockroach/pkg/storage.(*raftLogger).Panicf(0xc42072ea00, 0x192d563, 0x5d, 0xc420af31c0, 0x2, 0x2)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.go:111 +0x107
github.com/coreos/etcd/raft.(*raftLog).commitTo(0xc420aed0a0, 0x4de)
	/home/tamird/go/src/github.com/coreos/etcd/raft/log.go:191 +0x166
github.com/coreos/etcd/raft.(*raft).handleHeartbeat(0xc42037cb40, 0x8, 0x81, 0x1, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/coreos/etcd/raft/raft.go:1094 +0x54
github.com/coreos/etcd/raft.stepFollower(0xc42037cb40, 0x8, 0x81, 0x1, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/coreos/etcd/raft/raft.go:1040 +0x31f
github.com/coreos/etcd/raft.(*raft).Step(0xc42037cb40, 0x8, 0x81, 0x1, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/coreos/etcd/raft/raft.go:779 +0x11ef
github.com/coreos/etcd/raft.(*RawNode).Step(0xc420822a00, 0x8, 0x81, 0x1, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/coreos/etcd/raft/rawnode.go:195 +0xc9
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest.func4(0xc420822a00, 0x0, 0x0, 0x0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2948 +0x89
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroupLocked(0xc42088b800, 0x1ab3e00, 0xc420a57848, 0x6909ab, 0xc42088b8d8)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:509 +0x139
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).withRaftGroup(0xc42088b800, 0xc420a57848, 0x0, 0x0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:526 +0x92
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRaftRequest(0xc420094000, 0x7f530f21b878, 0xc420ef7c80, 0xc4209b9c80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2949 +0xb94
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc420094000, 0x1)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3177 +0x180
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420201680, 0xc42065c480)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:241 +0x267
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2()
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:181 +0x33
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42065c480, 0xc4207402a0)
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/home/tamird/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66


ERROR: exit status 2

1211 runs completed, 1 failures, over 16m38s
FAIL
make: *** [stress] Error 1
Makefile:128: recipe for target 'stress' failed