Last active
April 23, 2020 11:28
-
-
Save shashank855/bb304ae0d39832350df6d7851cf1ecf2 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| root@mlrack1:~# kubectl logs pvc-e069fd37-fa38-11e9-b8bb-0050569876a2-target-75f846c98c5w4kk -n openebs -ccstor-istgt | |
| * Starting enhanced syslogd rsyslogd | |
| ...done. | |
| Disabling dumping core | |
| 2020-04-15/13:36:13.294 main :3106: m#-1580050496.8 : istgt:0.5.20121028:04:15:25:Apr 14 2020: starting | |
| 2020-04-15/13:36:13.295 main :3154: m#-1580050496.8 : istgt:0.5.20121028:04:15:25:Apr 14 2020 read config mode:1 gcc-atomic | |
| 2020-04-15/13:36:25.213 istgt_reload :2158: u#10.139649213265664: reload configuration#1 start [istgt:0.5.20121028:04:15:25:Apr 14 2020] | |
| 2020-04-15/13:36:25.214 istgt_pg_update :2327: m#-1580050496.8 : update PG1 | |
| 2020-04-15/13:36:25.214 istgt_lu_add_unit :1792: u#10.139649213265664: trusty replica key {00AD628F7A3581F785B9FC2CD75935ED} and zvol guid {9737481031741255553} | |
| 2020-04-15/13:36:25.214 istgt_lu_add_unit :1792: u#10.139649213265664: trusty replica key {4D81FDCD71D9326B619B6679CA8E6636} and zvol guid {10152843271054983687} | |
| 2020-04-15/13:36:25.214 istgt_lu_add_unit :1792: u#10.139649213265664: trusty replica key {A98FE860792AFDADBEAB8FB10FC53B1C} and zvol guid {2332608751544743781} | |
| 2020-04-15/13:36:25.214 istgt_lu_disk_init: 932: u#10.139649213265664: lu_disk_init LU1 TargetName=iqn.2016-09.com.openebs.cstor:pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:25.215 istgt_lu_disk_init:1348: u#10.139649213265664: LU1: LUN0 iqn.2016-09.com.openebs.cstor:pvc-e069fd37-fa38-11e9-b8bb-0050569876a2: storage_online [(null), 20GB.0MB, 41943040 blks of 512 bytes, phy:3/6 WCE rpm:7200] q:32 thr:6/0 [0.0, 0.0] | |
| 2020-04-15/13:36:25.215 istgt_lu_disk_init:1365: u#10.139649213265664: LU1: LUN0 serial:e09ca7f6-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:25.216 istgt_lu_reload_up:3348: u#10.139649213265664: add LU1: Name=iqn.2016-09.com.openebs.cstor:pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:25.216 istgt_reload :2193: u#10.139649213265664: reload configuration#1 end [istgt:0.5.20121028:04:15:25:Apr 14 2020] | |
| 2020-04-15/13:36:25.971 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(10) (host=172.20.14.1 port=49374) | |
| 2020-04-15/13:36:25.974 update_replica_ent:1501: rm#-1610615040.8 : replica(9737481031741255553) connected successfully from 172.20.15.240:3232 rep: 3 | |
| 2020-04-15/13:36:26.713 worker :5944: c#0.139649091630848.: con:0/19 [10014ac:51980->172.21.253.64:3260,1] | |
| 2020-04-15/13:36:27.091 update_replica_ent:1653: rm#-1610615040.8 : replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-15/13:36:27.091 update_volstate :1189: rm#-1610615040.8 : Marking volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) not ready for IOs | |
| 2020-04-15/13:36:27.711 istgt_iscsi_op_log:1941: c#0.139649091630848.: login failed, target not ready | |
| 2020-04-15/13:36:27.713 worker :6125: c#0.139649091630848.: close conn events 25 | |
| 2020-04-15/13:36:27.713 sender :5911: s#0.139649083238144.: sender loop ended (0:19:51980) | |
| 2020-04-15/13:36:27.714 worker :6351: c#0.139649091630848.: worker 0/-1/51980 end (c#0.139649091630848/s#0.139649083238144) | |
| 2020-04-15/13:36:28.300 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(16) (host=172.20.10.1 port=51118) | |
| 2020-04-15/13:36:28.310 update_replica_ent:1501: rm#-1610615040.8 : replica(2332608751544743781) connected successfully from 172.20.11.37:3232 rep: 3 | |
| 2020-04-15/13:36:29.404 update_replica_ent:1653: rm#-1610615040.8 : replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-15/13:36:29.404 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22051190) healthy_replica(0) degraded_replica(2) | |
| 2020-04-15/13:36:29.715 worker :5944: c#1.139649053882112.: con:1/27 [10014ac:52086->172.21.253.64:3260,1] | |
| 2020-04-15/13:36:30.516 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(28) (host=172.20.12.1 port=48920) | |
| 2020-04-15/13:36:30.522 update_replica_ent:1501: rm#-1610615040.8 : replica(10152843271054983687) connected successfully from 172.20.12.180:3232 rep: 3 | |
| 2020-04-15/13:36:30.716 istgt_iscsi_drop_o:7227: c#1.139649053882112.: drop all 1 old connections 1 iqn.2016-09.com.openebs.cstor:pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 by iqn.1994-05.com.redhat:7b6afef7c533,i,0x00023d000001 | |
| 2020-04-15/13:36:30.717 istgt_iscsi_op_log:2380: c#1.139649053882112.: Login from iqn.1994-05.com.redhat:7b6afef7c533 (172.20.0.1) on iqn.2016-09.com.openebs.cstor:pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 LU1 (172.21.253.64:3260,1), ISID=23d000001, TSIH=1, CID=0, HeaderDigest=off, DataDigest=off | |
| 2020-04-15/13:36:31.633 update_replica_ent:1653: rm#-1610615040.8 : replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-15/13:36:31.633 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22051190) healthy_replica(0) degraded_replica(3) | |
| 2020-04-15/13:36:41.639 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:36:41.639 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(0) degraded count(3) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-15/13:36:41.640 trigger_rebuild : 711: rm#-1610615040.8 : Mesh rebuild will be attempted on replica(10152843271054983687) state:0 | |
| 2020-04-15/13:36:41.641 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:0 | |
| 2020-04-15/13:36:41.641 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-15/13:36:41.641 update_replica_sta:2861: rm#-1610615040.8 : Replica(10152843271054983687) state:0 rebuild status:0 | |
| 2020-04-15/13:36:41.641 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-15/13:36:41.643 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(10152843271054983687)state:0 | |
| 2020-04-15/13:36:41.643 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) state:0 | |
| 2020-04-15/13:36:51.650 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:36:51.650 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-15/13:36:51.651 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:0 | |
| 2020-04-15/13:36:51.651 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-15/13:36:51.651 update_replica_sta:2861: rm#-1610615040.8 : Replica(10152843271054983687) state:1 rebuild status:3 | |
| 2020-04-15/13:36:51.651 update_replica_sta:2878: rm#-1610615040.8 : Replica(10152843271054983687) (172.20.12.180:3232) mgmt_fd:28 state changed from degraded to healthy | |
| 2020-04-15/13:36:51.651 update_replica_sta:2971: rm#-1610615040.8 : Replica(10152843271054983687) marked healthy, seting master_replica to NULL | |
| 2020-04-15/13:36:51.651 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22051213) healthy_replica(1) degraded_replica(2) | |
| 2020-04-15/13:36:51.651 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(1) degraded count(2) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-15/13:36:51.651 trigger_rebuild : 711: rm#-1610615040.8 : Normal rebuild will be attempted on replica(2332608751544743781) state:0 | |
| 2020-04-15/13:36:51.654 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(2332608751544743781)state:0 | |
| 2020-04-15/13:36:51.654 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) state:0 | |
| 2020-04-15/13:37:01.662 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:37:01.662 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:01.662 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:2 | |
| 2020-04-15/13:37:01.662 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:11.670 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:37:11.670 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:11.671 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:2 | |
| 2020-04-15/13:37:11.671 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:21.683 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:37:21.683 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:21.684 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:2 | |
| 2020-04-15/13:37:21.684 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:31.696 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-15/13:37:31.696 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-15/13:37:31.697 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:1 rebuild status:3 | |
| 2020-04-15/13:37:31.697 update_replica_sta:2878: rm#-1610615040.8 : Replica(2332608751544743781) (172.20.11.37:3232) mgmt_fd:16 state changed from degraded to healthy | |
| 2020-04-15/13:37:31.697 update_replica_sta:2971: rm#-1610615040.8 : Replica(2332608751544743781) marked healthy, seting master_replica to NULL | |
| 2020-04-15/13:37:31.697 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22051237) healthy_replica(2) degraded_replica(1) | |
| 2020-04-15/13:37:31.697 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(2) degraded count(1) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-15/13:37:31.698 trigger_rebuild : 711: rm#-1610615040.8 : Normal rebuild will be attempted on replica(9737481031741255553) state:0 | |
| 2020-04-15/13:37:31.700 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(9737481031741255553)state:0 | |
| 2020-04-15/13:37:31.700 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) state:0 | |
| 2020-04-15/13:37:41.708 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:1 rebuild status:3 | |
| 2020-04-15/13:37:41.708 update_replica_sta:2878: rm#-1610615040.8 : Replica(9737481031741255553) (172.20.15.240:3232) mgmt_fd:10 state changed from degraded to healthy | |
| 2020-04-15/13:37:41.708 update_replica_sta:2971: rm#-1610615040.8 : Replica(9737481031741255553) marked healthy, seting master_replica to NULL | |
| 2020-04-15/13:37:41.708 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22051243) healthy_replica(3) degraded_replica(0) | |
| 2020-04-15/13:37:41.708 trigger_rebuild : 629: rm#-1610615040.8 : No downgraded in_quorum replicas on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2), rebuild will be attempted on non_quorum replicas | |
| 2020-04-15/13:37:41.708 trigger_rebuild : 630: rm#-1610615040.8 : Healthy count:(3) Degraded count:(0) Non-quorum-replica count:(0) | |
| 2020-04-15/13:37:41.708 trigger_rebuild : 634: rm#-1610615040.8 : No downgraded replicas on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2), rebuild will not be attempted | |
| 2020-04-21/22:00:26.613 istgt_lu_create_sn:2153: u#17.139649091630848: snap create ioseq: 22396099 resp: success | |
| 2020-04-21/22:09:26.330 istgt_lu_create_sn:2153: u#17.139649091630848: snap create ioseq: 22396387 resp: success | |
| 2020-04-22/02:52:15.038 replica_thread : 819: r#-1778391296.233260: Received event(8197) on fd(0) for replica(2332608751544743781) | |
| 2020-04-22/02:52:15.038 update_volstate :1186: r#-1778391296.233260: volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406723) healthy_replica(2) degraded_replica(0) | |
| 2020-04-22/02:52:15.038 respond_with_error: 274: r#-1778391296.233260: IO command set with error for replica(2332608751544743781) ..sent command(count:0 delay:0), queued command(count:0 delay:0), blocked command(count:0 delay:0).. read_error(0) write_error(0) | |
| 2020-04-22/02:52:15.038 handle_data_conn_e: 378: r#-1778391296.233260: Informing mgmt connection for error in replica(2332608751544743781) | |
| 2020-04-22/02:52:15.038 handle_mgmt_event_:4130: rm#-1610615040.8 : Got disconnect from data connection for replica(2332608751544743781) | |
| 2020-04-22/02:52:15.038 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) got disconnected from 172.20.11.37:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:15.039 replica_thread : 878: r#-1778391296.233260: replica_thread exiting ... | |
| 2020-04-22/02:52:15.049 replica_thread : 819: r#-1841301760.101528: Received event(8197) on fd(0) for replica(10152843271054983687) | |
| 2020-04-22/02:52:15.050 update_volstate :1189: r#-1841301760.101528: Marking volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) not ready for IOs | |
| 2020-04-22/02:52:15.050 respond_with_error: 274: r#-1841301760.101528: IO command set with error for replica(10152843271054983687) ..sent command(count:0 delay:0), queued command(count:0 delay:0), blocked command(count:0 delay:0).. read_error(0) write_error(0) | |
| 2020-04-22/02:52:15.050 handle_data_conn_e: 378: r#-1841301760.101528: Informing mgmt connection for error in replica(10152843271054983687) | |
| 2020-04-22/02:52:15.050 handle_mgmt_event_:4130: rm#-1610615040.8 : Got disconnect from data connection for replica(10152843271054983687) | |
| 2020-04-22/02:52:15.050 replica_thread : 878: r#-1841301760.101528: replica_thread exiting ... | |
| 2020-04-22/02:52:15.051 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) got disconnected from 172.20.12.180:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:15.056 replica_thread : 819: r#-1648363776.973748: Received event(8197) on fd(0) for replica(9737481031741255553) | |
| 2020-04-22/02:52:15.056 update_volstate :1189: r#-1648363776.973748: Marking volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) not ready for IOs | |
| 2020-04-22/02:52:15.059 respond_with_error: 274: r#-1648363776.973748: IO command set with error for replica(9737481031741255553) ..sent command(count:0 delay:0), queued command(count:0 delay:0), blocked command(count:0 delay:0).. read_error(0) write_error(0) | |
| 2020-04-22/02:52:15.059 handle_data_conn_e: 378: r#-1648363776.973748: Informing mgmt connection for error in replica(9737481031741255553) | |
| 2020-04-22/02:52:15.059 handle_mgmt_event_:4130: rm#-1610615040.8 : Got disconnect from data connection for replica(9737481031741255553) | |
| 2020-04-22/02:52:15.059 replica_thread : 878: r#-1648363776.973748: replica_thread exiting ... | |
| 2020-04-22/02:52:15.060 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) got disconnected from 172.20.15.240:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:15.972 istgt_lu_create_sn:2153: u#17.139649091630848: snap create ioseq: 22406723 resp: failed | |
| 2020-04-22/02:52:16.716 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(10) (host=172.20.12.1 port=36740) | |
| 2020-04-22/02:52:16.716 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(12) (host=172.20.14.1 port=37326) | |
| 2020-04-22/02:52:16.720 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(14) (host=172.20.10.1 port=44816) | |
| 2020-04-22/02:52:16.722 update_replica_ent:1501: rm#-1610615040.8 : replica(9737481031741255553) connected successfully from 172.20.15.240:3232 rep: 3 | |
| 2020-04-22/02:52:16.724 update_replica_ent:1526: rm#-1610615040.8 : data-open response is not OK for replica(9737481031741255553) | |
| 2020-04-22/02:52:16.728 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) got disconnected from 172.20.15.240:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:16.730 update_replica_ent:1501: rm#-1610615040.8 : replica(10152843271054983687) connected successfully from 172.20.12.180:3232 rep: 3 | |
| 2020-04-22/02:52:16.732 update_replica_ent:1526: rm#-1610615040.8 : data-open response is not OK for replica(10152843271054983687) | |
| 2020-04-22/02:52:16.733 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) got disconnected from 172.20.12.180:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:16.737 update_replica_ent:1501: rm#-1610615040.8 : replica(2332608751544743781) connected successfully from 172.20.11.37:3232 rep: 3 | |
| 2020-04-22/02:52:16.747 update_replica_ent:1526: rm#-1610615040.8 : data-open response is not OK for replica(2332608751544743781) | |
| 2020-04-22/02:52:16.747 handle_mgmt_conn_e:4068: rm#-1610615040.8 : Replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) got disconnected from 172.20.11.37:3232 mgmt_fd:-1 | |
| 2020-04-22/02:52:20.738 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(10) (host=172.20.14.1 port=37406) | |
| 2020-04-22/02:52:20.742 update_replica_ent:1501: rm#-1610615040.8 : replica(9737481031741255553) connected successfully from 172.20.15.240:3232 rep: 3 | |
| 2020-04-22/02:52:22.021 update_replica_ent:1653: rm#-1610615040.8 : replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-22/02:52:22.021 update_volstate :1189: rm#-1610615040.8 : Marking volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) not ready for IOs | |
| 2020-04-22/02:52:22.021 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(16) (host=172.20.12.1 port=36842) | |
| 2020-04-22/02:52:22.021 accept_mgmt_conns :3079: rm#-1610615040.8 : Accepted connection on descriptor(18) (host=172.20.10.1 port=44920) | |
| 2020-04-22/02:52:22.026 update_replica_ent:1501: rm#-1610615040.8 : replica(10152843271054983687) connected successfully from 172.20.12.180:3232 rep: 3 | |
| 2020-04-22/02:52:23.170 update_replica_ent:1653: rm#-1610615040.8 : replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-22/02:52:23.170 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406723) healthy_replica(0) degraded_replica(2) | |
| 2020-04-22/02:52:23.182 update_replica_ent:1501: rm#-1610615040.8 : replica(2332608751544743781) connected successfully from 172.20.11.37:3232 rep: 3 | |
| 2020-04-22/02:52:24.390 update_replica_ent:1653: rm#-1610615040.8 : replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) connected to target with needs_update: 0 can_be_trusty: 1 | |
| 2020-04-22/02:52:24.390 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406723) healthy_replica(0) degraded_replica(3) | |
| 2020-04-22/02:52:34.401 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-22/02:52:34.401 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(0) degraded count(3) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-22/02:52:34.401 trigger_rebuild : 711: rm#-1610615040.8 : Mesh rebuild will be attempted on replica(2332608751544743781) state:0 | |
| 2020-04-22/02:52:34.402 update_replica_sta:2861: rm#-1610615040.8 : Replica(10152843271054983687) state:0 rebuild status:0 | |
| 2020-04-22/02:52:34.402 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-22/02:52:34.403 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:0 rebuild status:0 | |
| 2020-04-22/02:52:34.403 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-22/02:52:34.405 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(2332608751544743781)state:0 | |
| 2020-04-22/02:52:34.405 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(A98FE860792AFDADBEAB8FB10FC53B1C:2332608751544743781) state:0 | |
| 2020-04-22/02:52:44.422 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-22/02:52:44.422 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (2332608751544743781) | |
| 2020-04-22/02:52:44.423 update_replica_sta:2861: rm#-1610615040.8 : Replica(2332608751544743781) state:1 rebuild status:3 | |
| 2020-04-22/02:52:44.423 update_replica_sta:2878: rm#-1610615040.8 : Replica(2332608751544743781) (172.20.11.37:3232) mgmt_fd:18 state changed from degraded to healthy | |
| 2020-04-22/02:52:44.423 update_replica_sta:2971: rm#-1610615040.8 : Replica(2332608751544743781) marked healthy, seting master_replica to NULL | |
| 2020-04-22/02:52:44.423 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406739) healthy_replica(1) degraded_replica(2) | |
| 2020-04-22/02:52:44.423 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(1) degraded count(2) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-22/02:52:44.423 trigger_rebuild : 711: rm#-1610615040.8 : Normal rebuild will be attempted on replica(10152843271054983687) state:0 | |
| 2020-04-22/02:52:44.423 update_replica_sta:2861: rm#-1610615040.8 : Replica(10152843271054983687) state:0 rebuild status:0 | |
| 2020-04-22/02:52:44.423 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-22/02:52:44.426 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(10152843271054983687)state:0 | |
| 2020-04-22/02:52:44.426 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(4D81FDCD71D9326B619B6679CA8E6636:10152843271054983687) state:0 | |
| 2020-04-22/02:52:54.451 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:0 rebuild status:0 | |
| 2020-04-22/02:52:54.451 trigger_rebuild : 622: rm#-1610615040.8 : Rebuild is already in progress on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) for replica (10152843271054983687) | |
| 2020-04-22/02:52:54.452 update_replica_sta:2861: rm#-1610615040.8 : Replica(10152843271054983687) state:1 rebuild status:3 | |
| 2020-04-22/02:52:54.452 update_replica_sta:2878: rm#-1610615040.8 : Replica(10152843271054983687) (172.20.12.180:3232) mgmt_fd:16 state changed from degraded to healthy | |
| 2020-04-22/02:52:54.452 update_replica_sta:2971: rm#-1610615040.8 : Replica(10152843271054983687) marked healthy, seting master_replica to NULL | |
| 2020-04-22/02:52:54.452 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406744) healthy_replica(2) degraded_replica(1) | |
| 2020-04-22/02:52:54.452 trigger_rebuild : 704: rm#-1610615040.8 : Healthy count(2) degraded count(1) consistency factor(2) replication factor(3) desired replication factor(3) | |
| 2020-04-22/02:52:54.452 trigger_rebuild : 711: rm#-1610615040.8 : Normal rebuild will be attempted on replica(9737481031741255553) state:0 | |
| 2020-04-22/02:52:54.456 start_rebuild : 539: rm#-1610615040.8 : start_rebuild opcode sent for Replica(9737481031741255553)state:0 | |
| 2020-04-22/02:52:54.456 handle_prepare_for:2621: rm#-1610615040.8 : Rebuild triggered on Replica(00AD628F7A3581F785B9FC2CD75935ED:9737481031741255553) state:0 | |
| 2020-04-22/02:53:04.487 update_replica_sta:2861: rm#-1610615040.8 : Replica(9737481031741255553) state:1 rebuild status:3 | |
| 2020-04-22/02:53:04.487 update_replica_sta:2878: rm#-1610615040.8 : Replica(9737481031741255553) (172.20.15.240:3232) mgmt_fd:10 state changed from degraded to healthy | |
| 2020-04-22/02:53:04.487 update_replica_sta:2971: rm#-1610615040.8 : Replica(9737481031741255553) marked healthy, seting master_replica to NULL | |
| 2020-04-22/02:53:04.487 update_volstate :1186: rm#-1610615040.8 : volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) is ready for IOs now.. io_seq(22406749) healthy_replica(3) degraded_replica(0) | |
| 2020-04-22/02:53:04.488 trigger_rebuild : 629: rm#-1610615040.8 : No downgraded in_quorum replicas on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2), rebuild will be attempted on non_quorum replicas | |
| 2020-04-22/02:53:04.488 trigger_rebuild : 630: rm#-1610615040.8 : Healthy count:(3) Degraded count:(0) Non-quorum-replica count:(0) | |
| 2020-04-22/02:53:04.488 trigger_rebuild : 634: rm#-1610615040.8 : No downgraded replicas on volume(pvc-e069fd37-fa38-11e9-b8bb-0050569876a2), rebuild will not be attempted | |
| root@mlrack1:~# kubectl logs cstor-disk-gitlab-0btn-6b6b565f94-lr9zp -n openebs -ccstor-pool | |
| Disabling dumping core | |
| sleeping for 2 sec | |
| 2020-04-15/13:16:03.832 disabled auto import (reading of zpool.cache) | |
| physmem = 1048576 pages (4.00 GB) | |
| 2020-04-15/13:16:23.380 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.380 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.381 Instantiating zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:16:23.386 [tgt 172.21.253.64:6060:17]: Connected | |
| 2020-04-15/13:16:23.386 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.387 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.387 [tgt 172.21.253.64:6060:17]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:16:23.461 Instantiating zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.470 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.470 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.518 Instantiating zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.526 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.526 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.527 Instantiating zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.640 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:16:23.640 IO sequence number:21001148 Degraded IO sequence number:20928178 | |
| 2020-04-15/13:16:23.642 [tgt 172.21.27.190:6060:19]: Connected | |
| 2020-04-15/13:16:23.642 [tgt 172.21.24.254:6060:18]: Connected | |
| 2020-04-15/13:16:23.643 [tgt 172.21.133.201:6060:20]: Connected | |
| 2020-04-15/13:16:23.643 [tgt 172.21.27.190:6060:19]: Handshake command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.645 New data connection on fd 21 | |
| 2020-04-15/13:16:23.646 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:16:23.646 IO sequence number:35790439 Degraded IO sequence number:35750118 | |
| 2020-04-15/13:16:23.646 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap | |
| 2020-04-15/13:16:23.646 [tgt 172.21.133.201:6060:20]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.649 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:16:23.649 IO sequence number:69140470 Degraded IO sequence number:69060882 | |
| 2020-04-15/13:16:23.649 [tgt 172.21.24.254:6060:18]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:23.650 New data connection on fd 22 | |
| 2020-04-15/13:16:23.654 ERROR fail on unavailable snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:16:23.654 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:16:23.654 IO sequence number:40346593 Degraded IO sequence number:40292436 | |
| 2020-04-15/13:16:23.663 New data connection on fd 8 | |
| 2020-04-15/13:16:23.664 New data connection on fd 24 | |
| 2020-04-15/13:16:23.664 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:16:23.666 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:16:23.776 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.777 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.777 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.777 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.777 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.777 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.785 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.785 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.828 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.828 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.833 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.834 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.836 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 21 | |
| 2020-04-15/13:16:23.836 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 21 | |
| 2020-04-15/13:16:23.839 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 24 | |
| 2020-04-15/13:16:23.839 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 24 | |
| 2020-04-15/13:16:23.892 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.892 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.893 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:23.893 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:23.897 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 22 | |
| 2020-04-15/13:16:23.897 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 22 | |
| 2020-04-15/13:16:23.898 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 8 | |
| 2020-04-15/13:16:23.898 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 8 | |
| 2020-04-15/13:16:34.841 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:16:34.843 [tgt 172.21.24.254:6060:18]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:34.846 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.847 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:14] at 172.20.12.168:3233 helping in rebuild | |
| 2020-04-15/13:16:34.849 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.849 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:25] at 172.20.12.168:3233 helping in rebuild | |
| 2020-04-15/13:16:34.855 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.856 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.856 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:16:34.856 IO sequence number:40346593 Degraded IO sequence number:40292436 | |
| 2020-04-15/13:16:34.856 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:26] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:16:34.856 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(25) | |
| 2020-04-15/13:16:34.856 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:16:34.857 IO sequence number:21001148 Degraded IO sequence number:21002013 | |
| 2020-04-15/13:16:34.857 New rebuild connection on fd 27 | |
| 2020-04-15/13:16:34.857 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:29] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:16:34.857 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(14) | |
| 2020-04-15/13:16:34.858 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(27) | |
| 2020-04-15/13:16:34.858 New rebuild connection on fd 28 | |
| 2020-04-15/13:16:34.858 Checkpointed IO_seq: 40346593, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:34.858 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(28) | |
| 2020-04-15/13:16:34.859 Checkpointed IO_seq: 21001148, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:16:34.908 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:34.908 [tgt 172.21.27.190:6060:19]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:34.911 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586956594 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:34.912 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586956594 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:34.912 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.912 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:30] at 172.20.15.229:3233 helping in rebuild | |
| 2020-04-15/13:16:34.913 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.913 [pvc-7193bd1f-3457-11e9-816f-0050569876a2:31] at 172.20.12.168:3233 helping in rebuild | |
| 2020-04-15/13:16:34.919 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.920 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:16:34.920 IO sequence number:69140470 Degraded IO sequence number:69142253 | |
| 2020-04-15/13:16:34.920 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:32] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:16:34.921 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(30) | |
| 2020-04-15/13:16:34.927 New rebuild connection on fd 33 | |
| 2020-04-15/13:16:34.928 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(33) | |
| 2020-04-15/13:16:34.928 Checkpointed IO_seq: 69140470, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:34.935 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:16:34.936 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:16:34.936 IO sequence number:35790439 Degraded IO sequence number:35750118 | |
| 2020-04-15/13:16:34.936 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2:34] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:16:34.937 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 on fd(31) | |
| 2020-04-15/13:16:34.943 New rebuild connection on fd 35 | |
| 2020-04-15/13:16:34.944 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(35) | |
| 2020-04-15/13:16:34.944 Checkpointed IO_seq: 35790439, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:34.947 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:34.947 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:34.947 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-15/13:16:34.975 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586956594 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:34.975 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586956594 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:35.031 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586956594 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:35.031 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586956594 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:35.036 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586956594 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:16:35.036 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586956594 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:16:37.643 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:16:38.345 Checkpointed IO_seq: 35790439, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:41.162 Checkpointed IO_seq: 35790439, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:43.437 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-15/13:16:44.541 Checkpointed IO_seq: 35790439, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:44.858 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:16:44.865 [tgt 172.21.24.254:6060:18]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:44.923 [tgt 172.21.27.190:6060:19]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:44.938 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:50.600 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:16:51.999 Checkpointed IO_seq: 35790439, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:16:54.879 [tgt 172.21.24.254:6060:18]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:54.881 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:16:54.937 [tgt 172.21.27.190:6060:19]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:54.961 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:55.828 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:16:55.829 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:16:55.829 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:16:55.830 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586956594 | |
| 2020-04-15/13:16:55.924 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:55.924 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(27) | |
| 2020-04-15/13:16:55.930 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:55.930 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:55.932 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:16:56.403 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:16:56.403 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:16:56.404 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:16:56.403 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586956594 | |
| 2020-04-15/13:16:56.478 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:56.478 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(35) | |
| 2020-04-15/13:16:56.481 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:56.481 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:16:56.482 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:17:02.986 Checkpointed IO_seq: 69140470, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:17:04.893 [tgt 172.21.24.254:6060:18]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:04.894 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:04.949 [tgt 172.21.27.190:6060:19]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:04.975 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:12.928 Checkpointed IO_seq: 69140470, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:17:14.907 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:14.989 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:21.164 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:17:23.375 Checkpointed IO_seq: 69140470, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:17:25.004 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:28.183 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:17:28.183 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586956594 | |
| 2020-04-15/13:17:28.186 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:17:28.186 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:17:28.236 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:28.236 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(33) | |
| 2020-04-15/13:17:28.304 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:28.304 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:28.308 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:17:33.533 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:35.030 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:17:40.787 Checkpointed IO_seq: 21001148, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:17:43.549 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:43.801 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-15/13:17:43.802 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:17:43.802 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:17:43.802 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586956594 | |
| 2020-04-15/13:17:43.847 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:43.847 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(28) | |
| 2020-04-15/13:17:43.848 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:43.848 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:17:43.849 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:17:53.570 [tgt 172.21.253.64:6060:17]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:20:45.947 [tgt 172.21.27.190:6060:19]: Prepare for rebuild command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:20:45.948 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:20:45.948 IO sequence number:35790439 Degraded IO sequence number:35790831 | |
| 2020-04-15/13:20:45.953 New rebuild connection on fd 23 | |
| 2020-04-15/13:20:45.954 Rebuild scanner started on zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(23) | |
| 2020-04-15/13:20:45.955 Checkpointed IO_seq: 35790706, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:20:45.958 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:20:45.958 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:20:45.958 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-15/13:20:46.008 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790706.1586956845 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:20:46.008 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790706.1586956845 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:20:47.867 Checkpointed IO_seq: 35790706, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:20:49.560 Checkpointed IO_seq: 35790706, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:20:50.834 Checkpointed IO_seq: 35790706, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:20:54.099 Checkpointed IO_seq: 35790706, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:20:57.570 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790706.1586956845 | |
| 2020-04-15/13:20:57.827 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:20:57.827 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(23) | |
| 2020-04-15/13:22:56.676 [tgt 172.21.253.64:6060:17]: Prepare for rebuild command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:22:56.676 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:22:56.676 IO sequence number:21001148 Degraded IO sequence number:21002049 | |
| 2020-04-15/13:22:56.686 New rebuild connection on fd 14 | |
| 2020-04-15/13:22:56.686 Rebuild scanner started on zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(14) | |
| 2020-04-15/13:22:56.686 Checkpointed IO_seq: 21001690, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:22:56.762 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21001690.1586956976 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:22:56.762 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21001690.1586956976 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:22:56.817 [tgt 172.21.133.201:6060:20]: Prepare for rebuild command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:22:56.818 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:22:56.818 IO sequence number:69140470 Degraded IO sequence number:69142305 | |
| 2020-04-15/13:22:56.821 [tgt 172.21.24.254:6060:18]: Prepare for rebuild command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:22:56.822 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:22:56.822 IO sequence number:40346593 Degraded IO sequence number:40347159 | |
| 2020-04-15/13:22:56.822 New rebuild connection on fd 25 | |
| 2020-04-15/13:22:56.823 Rebuild scanner started on zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(25) | |
| 2020-04-15/13:22:56.823 Checkpointed IO_seq: 69140511, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:22:56.829 New rebuild connection on fd 26 | |
| 2020-04-15/13:22:56.831 Rebuild scanner started on zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-15/13:22:56.831 Checkpointed IO_seq: 40346880, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:22:56.864 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586956976 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:22:56.864 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586956976 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:22:56.933 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40346880.1586956976 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:22:56.933 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40346880.1586956976 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:23:00.277 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40346880.1586956976 | |
| 2020-04-15/13:23:00.452 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:23:00.453 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-15/13:23:00.847 Checkpointed IO_seq: 69140511, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:23:01.107 Checkpointed IO_seq: 21001690, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:23:02.805 Checkpointed IO_seq: 69140511, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:23:02.982 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21001690.1586956976 | |
| 2020-04-15/13:23:03.044 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:23:03.044 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(14) | |
| 2020-04-15/13:23:08.152 Checkpointed IO_seq: 69140511, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:23:11.535 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586956976 | |
| 2020-04-15/13:23:11.586 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:23:11.586 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(25) | |
| 2020-04-15/13:29:47.388 Connection closed by the peer for socket(8) | |
| 2020-04-15/13:29:47.389 Waiting for conn_closed(1) and ack_sender (-1)for fd(8) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:29:47.389 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 closed on fd: 8 | |
| 2020-04-15/13:29:47.390 ERROR connection closed for fd(20) | |
| 2020-04-15/13:29:47.390 [tgt 172.21.133.201:6060:20]: Closing the connection | |
| 2020-04-15/13:29:49.389 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-15/13:29:49.389 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-15/13:29:49.389 Data connection closed on fd: 8 | |
| 2020-04-15/13:29:50.397 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:29:53.403 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:02.323 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:30:05.329 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:30:12.337 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:30:15.344 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:30:22.341 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:30:25.347 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:32.359 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:37.367 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:42.375 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:47.393 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:50.399 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:30:57.409 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:31:00.415 ERROR Failed to connect to 172.21.133.201:6060 fd(8) | |
| 2020-04-15/13:31:07.425 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:31:10.431 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:31:17.443 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:31:20.449 ERROR Failed to connect to 172.21.133.201:6060 fd(14) | |
| 2020-04-15/13:31:24.457 [tgt 172.21.133.201:6060:20]: Connected | |
| 2020-04-15/13:31:25.476 [tgt 172.21.133.201:6060:20]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:25.477 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:31:25.478 IO sequence number:69140470 Degraded IO sequence number:69143427 | |
| 2020-04-15/13:31:26.821 New data connection on fd 14 | |
| 2020-04-15/13:31:26.824 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:31:26.894 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:31:26.894 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:31:26.944 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:31:26.944 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:31:26.950 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 14 | |
| 2020-04-15/13:31:26.950 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 14 | |
| 2020-04-15/13:31:37.963 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:37.967 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:31:37.967 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:23] at 172.20.2.87:3233 helping in rebuild | |
| 2020-04-15/13:31:37.971 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:31:37.972 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:31:37.972 IO sequence number:69140470 Degraded IO sequence number:70192004 | |
| 2020-04-15/13:31:37.972 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:25] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:31:37.973 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(23) | |
| 2020-04-15/13:31:37.973 New rebuild connection on fd 26 | |
| 2020-04-15/13:31:37.974 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-15/13:31:37.974 Checkpointed IO_seq: 69140470, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:38.034 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586957497 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:31:38.034 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586957497 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:31:40.224 Checkpointed IO_seq: 69140470, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:41.818 Checkpointed IO_seq: 69140470, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:44.746 Checkpointed IO_seq: 69140470, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:47.234 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:31:47.234 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140470.1586957497 | |
| 2020-04-15/13:31:47.283 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.283 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-15/13:31:47.627 ERROR connection closed for fd(18) | |
| 2020-04-15/13:31:47.627 Connection closed by the peer for socket(24) | |
| 2020-04-15/13:31:47.627 [tgt 172.21.24.254:6060:18]: Closing the connection | |
| 2020-04-15/13:31:47.628 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 closed on fd: 24 | |
| 2020-04-15/13:31:47.627 Waiting for conn_closed(1) and ack_sender (-1)for fd(24) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.814 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:31:47.815 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:31:47.815 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:31:47.892 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.892 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.894 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:31:47.979 [tgt 172.21.133.201:6060:20]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.982 [tgt 172.21.133.201:6060:20]: Prepare for rebuild command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:31:47.982 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-15/13:31:47.982 IO sequence number:69140470 Degraded IO sequence number:70192004 | |
| 2020-04-15/13:31:47.987 New rebuild connection on fd 23 | |
| 2020-04-15/13:31:47.988 Rebuild scanner started on zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(23) | |
| 2020-04-15/13:31:47.988 Checkpointed IO_seq: 69140511, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:48.068 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586957507 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:31:48.068 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586957507 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:31:49.631 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-15/13:31:49.631 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-15/13:31:49.631 Data connection closed on fd: 24 | |
| 2020-04-15/13:31:50.269 Checkpointed IO_seq: 69140511, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:50.985 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:31:51.958 Checkpointed IO_seq: 69140511, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:53.991 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:31:56.807 Checkpointed IO_seq: 69140511, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:31:59.999 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69140511.1586957507 | |
| 2020-04-15/13:32:00.099 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:32:00.099 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(23) | |
| 2020-04-15/13:32:01.013 ERROR Failed to connect to 172.21.24.254:6060 fd(8) | |
| 2020-04-15/13:32:04.010 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:13.015 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:18.023 ERROR Failed to connect to 172.21.24.254:6060 fd(8) | |
| 2020-04-15/13:32:23.033 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:26.039 ERROR Failed to connect to 172.21.24.254:6060 fd(8) | |
| 2020-04-15/13:32:33.045 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:36.051 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:43.050 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:46.057 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:53.061 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:32:56.068 ERROR Failed to connect to 172.21.24.254:6060 fd(8) | |
| 2020-04-15/13:33:03.070 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:33:06.075 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:33:13.085 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:33:16.092 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:33:23.103 ERROR Failed to connect to 172.21.24.254:6060 fd(8) | |
| 2020-04-15/13:33:28.111 ERROR Failed to connect to 172.21.24.254:6060 fd(18) | |
| 2020-04-15/13:33:30.115 [tgt 172.21.24.254:6060:23]: Connected | |
| 2020-04-15/13:33:30.596 [tgt 172.21.24.254:6060:23]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:30.596 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:33:30.597 IO sequence number:40346593 Degraded IO sequence number:40347553 | |
| 2020-04-15/13:33:30.599 New data connection on fd 8 | |
| 2020-04-15/13:33:30.601 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:33:30.676 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:33:30.676 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:33:30.746 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:33:30.746 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:33:30.752 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 8 | |
| 2020-04-15/13:33:30.752 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 8 | |
| 2020-04-15/13:33:39.374 Connection closed by the peer for socket(22) | |
| 2020-04-15/13:33:39.375 Waiting for conn_closed(1) and ack_sender (-1)for fd(22) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:39.375 ERROR connection closed for fd(19) | |
| 2020-04-15/13:33:39.375 [tgt 172.21.27.190:6060:19]: Closing the connection | |
| 2020-04-15/13:33:39.375 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 closed on fd: 22 | |
| 2020-04-15/13:33:41.378 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-15/13:33:41.378 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-15/13:33:41.378 Data connection closed on fd: 22 | |
| 2020-04-15/13:33:41.762 [tgt 172.21.24.254:6060:23]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:41.767 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:33:41.768 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:19] at 172.20.15.240:3233 helping in rebuild | |
| 2020-04-15/13:33:41.772 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:33:41.772 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:33:41.772 IO sequence number:40346593 Degraded IO sequence number:41396152 | |
| 2020-04-15/13:33:41.772 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:22] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:33:41.772 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(19) | |
| 2020-04-15/13:33:41.772 New rebuild connection on fd 25 | |
| 2020-04-15/13:33:41.773 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(25) | |
| 2020-04-15/13:33:41.773 Checkpointed IO_seq: 40346593, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:33:41.809 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:33:41.809 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:33:42.382 ERROR Failed to connect to 172.21.27.190:6060 fd(18) | |
| 2020-04-15/13:33:42.992 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:33:42.993 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621 | |
| 2020-04-15/13:33:43.323 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:43.324 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(25) | |
| 2020-04-15/13:33:43.485 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:33:43.486 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:33:43.486 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:33:43.550 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:43.550 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:43.557 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:33:45.387 ERROR Failed to connect to 172.21.27.190:6060 fd(18) | |
| 2020-04-15/13:33:51.779 [tgt 172.21.24.254:6060:23]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:51.782 [tgt 172.21.24.254:6060:23]: Prepare for rebuild command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:51.783 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-15/13:33:51.783 IO sequence number:40346593 Degraded IO sequence number:41396156 | |
| 2020-04-15/13:33:51.788 New rebuild connection on fd 24 | |
| 2020-04-15/13:33:51.789 Rebuild scanner started on zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(24) | |
| 2020-04-15/13:33:51.789 Checkpointed IO_seq: 40347213, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:33:51.884 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40347213.1586957631 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:33:51.884 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40347213.1586957631 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:33:53.129 ERROR Failed to connect to 172.21.27.190:6060 fd(18) | |
| 2020-04-15/13:33:53.578 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap40347213.1586957631 | |
| 2020-04-15/13:33:53.784 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:33:53.785 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(24) | |
| 2020-04-15/13:33:56.135 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:03.141 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:06.147 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:13.154 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:16.163 ERROR Failed to connect to 172.21.27.190:6060 fd(22) | |
| 2020-04-15/13:34:23.157 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:26.163 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:33.177 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:36.186 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:43.193 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:48.204 ERROR Failed to connect to 172.21.27.190:6060 fd(19) | |
| 2020-04-15/13:34:51.207 ERROR Failed to connect to 172.21.27.190:6060 fd(22) | |
| 2020-04-15/13:34:53.636 Connection closed by the peer for socket(21) | |
| 2020-04-15/13:34:53.636 ERROR connection closed for fd(17) | |
| 2020-04-15/13:34:53.636 [tgt 172.21.253.64:6060:17]: Closing the connection | |
| 2020-04-15/13:34:53.636 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 closed on fd: 21 | |
| 2020-04-15/13:34:53.636 Waiting for conn_closed(1) and ack_sender (-1)for fd(21) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:34:55.638 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-15/13:34:55.638 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-15/13:34:55.638 Data connection closed on fd: 21 | |
| 2020-04-15/13:34:56.641 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:34:58.646 ERROR Failed to connect to 172.21.27.190:6060 fd(18) | |
| 2020-04-15/13:34:59.649 ERROR Failed to connect to 172.21.253.64:6060 fd(19) | |
| 2020-04-15/13:35:01.653 ERROR Failed to connect to 172.21.27.190:6060 fd(17) | |
| 2020-04-15/13:35:05.875 [tgt 172.21.27.190:6060:18]: Connected | |
| 2020-04-15/13:35:06.717 [tgt 172.21.27.190:6060:18]: Handshake command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:06.718 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:35:06.718 IO sequence number:35790439 Degraded IO sequence number:35791038 | |
| 2020-04-15/13:35:06.723 New data connection on fd 19 | |
| 2020-04-15/13:35:06.724 ERROR fail on unavailable snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-15/13:35:06.746 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:06.747 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:06.773 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:06.773 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:06.775 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 19 | |
| 2020-04-15/13:35:06.775 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 19 | |
| 2020-04-15/13:35:06.877 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:35:09.883 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:35:16.885 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:35:19.891 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:35:21.520 [tgt 172.21.27.190:6060:18]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:26.899 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:35:29.905 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:35:31.531 [tgt 172.21.27.190:6060:18]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:31.536 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:35:31.536 [pvc-7193bd1f-3457-11e9-816f-0050569876a2:21] at 172.20.2.87:3233 helping in rebuild | |
| 2020-04-15/13:35:31.541 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:35:31.542 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:35:31.542 IO sequence number:35790439 Degraded IO sequence number:35791038 | |
| 2020-04-15/13:35:31.543 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2:22] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:35:31.543 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 on fd(21) | |
| 2020-04-15/13:35:31.543 New rebuild connection on fd 24 | |
| 2020-04-15/13:35:31.544 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(24) | |
| 2020-04-15/13:35:31.544 Checkpointed IO_seq: 35790439, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:31.546 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:31.546 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:31.546 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-15/13:35:31.590 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586957731 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:31.591 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586957731 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:33.164 Checkpointed IO_seq: 35790439, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:33.915 Checkpointed IO_seq: 35790439, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:34.996 Checkpointed IO_seq: 35790439, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:36.909 ERROR Failed to connect to 172.21.253.64:6060 fd(26) | |
| 2020-04-15/13:35:37.280 Checkpointed IO_seq: 35790439, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:39.049 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:35:39.241 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-15/13:35:39.241 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap35790439.1586957731 | |
| 2020-04-15/13:35:39.242 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:35:39.242 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:35:39.268 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:39.268 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(24) | |
| 2020-04-15/13:35:39.269 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:39.269 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:39.269 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:35:39.915 ERROR Failed to connect to 172.21.253.64:6060 fd(25) | |
| 2020-04-15/13:35:41.547 [tgt 172.21.27.190:6060:18]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:41.549 [tgt 172.21.27.190:6060:18]: Prepare for rebuild command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:41.551 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-15/13:35:41.551 IO sequence number:35790439 Degraded IO sequence number:35791038 | |
| 2020-04-15/13:35:41.554 New rebuild connection on fd 17 | |
| 2020-04-15/13:35:41.554 Rebuild scanner started on zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(17) | |
| 2020-04-15/13:35:41.554 Checkpointed IO_seq: 35790864, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:41.557 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:41.557 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:41.557 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-15/13:35:41.607 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790864.1586957741 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:35:41.607 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790864.1586957741 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:35:42.746 Checkpointed IO_seq: 35790864, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:43.808 Checkpointed IO_seq: 35790864, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:44.945 Checkpointed IO_seq: 35790864, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:46.367 Checkpointed IO_seq: 35790864, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-15/13:35:46.917 ERROR Failed to connect to 172.21.253.64:6060 fd(22) | |
| 2020-04-15/13:35:47.951 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap35790864.1586957741 | |
| 2020-04-15/13:35:48.219 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-15/13:35:48.220 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(17) | |
| 2020-04-15/13:35:49.923 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:35:56.925 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:35:59.931 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:36:06.939 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:36:09.945 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:36:16.949 ERROR Failed to connect to 172.21.253.64:6060 fd(17) | |
| 2020-04-15/13:36:19.955 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:36:26.961 ERROR Failed to connect to 172.21.253.64:6060 fd(21) | |
| 2020-04-15/13:36:28.293 [tgt 172.21.253.64:6060:21]: Connected | |
| 2020-04-15/13:36:28.294 [tgt 172.21.253.64:6060:21]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:28.295 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:36:28.295 IO sequence number:21001148 Degraded IO sequence number:21002614 | |
| 2020-04-15/13:36:28.299 New data connection on fd 22 | |
| 2020-04-15/13:36:28.305 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap | |
| 2020-04-15/13:36:28.349 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:36:28.349 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-15/13:36:28.394 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:36:28.394 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-15/13:36:28.396 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 22 | |
| 2020-04-15/13:36:28.396 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 22 | |
| 2020-04-15/13:36:41.632 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:51.643 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:51.648 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-15/13:36:51.649 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:17] at 172.20.12.180:3233 helping in rebuild | |
| 2020-04-15/13:36:51.652 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-15/13:36:51.653 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:36:51.653 IO sequence number:21001148 Degraded IO sequence number:22051207 | |
| 2020-04-15/13:36:51.653 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:25] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-15/13:36:51.653 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(17) | |
| 2020-04-15/13:36:51.654 New rebuild connection on fd 26 | |
| 2020-04-15/13:36:51.655 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26) | |
| 2020-04-15/13:36:51.655 Checkpointed IO_seq: 21001148, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:36:51.722 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586957811 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:36:51.722 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586957811 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:36:56.133 Checkpointed IO_seq: 21001148, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:36:57.999 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-15/13:36:58.000 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001148.1586957811 | |
| 2020-04-15/13:36:58.376 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:36:58.376 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26) | |
| 2020-04-15/13:37:01.654 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:11.663 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:21.676 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:21.869 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-15/13:37:21.870 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-15/13:37:21.870 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-15/13:37:21.911 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:21.911 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:21.912 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-15/13:37:31.690 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:31.692 [tgt 172.21.253.64:6060:21]: Prepare for rebuild command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:31.693 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-15/13:37:31.693 IO sequence number:21001148 Degraded IO sequence number:22051226 | |
| 2020-04-15/13:37:31.696 New rebuild connection on fd 24 | |
| 2020-04-15/13:37:31.697 Rebuild scanner started on zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(24) | |
| 2020-04-15/13:37:31.697 Checkpointed IO_seq: 21002229, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:37:31.763 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21002229.1586957851 status change: DEGRADED -> DEGRADED | |
| 2020-04-15/13:37:31.763 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21002229.1586957851 rebuild status change: INIT -> INIT | |
| 2020-04-15/13:37:33.556 Checkpointed IO_seq: 21002229, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-15/13:37:34.489 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap21002229.1586957851 | |
| 2020-04-15/13:37:34.530 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-15/13:37:34.530 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(24) | |
| 2020-04-21/21:59:49.152 [tgt 172.21.27.190:6060:18]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:00:21.535 [tgt 172.21.133.201:6060:20]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:00:25.611 [tgt 172.21.253.64:6060:21]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup | |
| 2020-04-21/22:00:27.678 [tgt 172.21.24.254:6060:23]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:09:09.506 [tgt 172.21.27.190:6060:18]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:09:21.385 [tgt 172.21.133.201:6060:20]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:09:25.325 [tgt 172.21.253.64:6060:21]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:09:27.363 [tgt 172.21.24.254:6060:23]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:19:02.830 [tgt 172.21.27.190:6060:18]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:19:02.901 ERROR Failed to create cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup: 17 | |
| 2020-04-21/22:19:02.901 Connection closed by the peer for socket(19) | |
| 2020-04-21/22:19:02.901 ERROR [tgt 172.21.27.190:6060:18]: Error reply with status 1 for OP 14 | |
| 2020-04-21/22:19:02.902 Waiting for conn_closed(1) and ack_sender (-1)for fd(19) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:02.902 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 closed on fd: 19 | |
| 2020-04-21/22:19:02.904 ERROR connection closed for fd(18) | |
| 2020-04-21/22:19:02.905 [tgt 172.21.27.190:6060:18]: Closing the connection | |
| 2020-04-21/22:19:04.506 [tgt 172.21.133.201:6060:20]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:19:04.543 ERROR Failed to create cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup: 17 | |
| 2020-04-21/22:19:04.543 Connection closed by the peer for socket(14) | |
| 2020-04-21/22:19:04.543 Waiting for conn_closed(1) and ack_sender (-1)for fd(14) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:04.544 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 closed on fd: 14 | |
| 2020-04-21/22:19:04.545 ERROR [tgt 172.21.133.201:6060:20]: Error reply with status 1 for OP 14 | |
| 2020-04-21/22:19:04.547 [tgt 172.21.27.190:6060:18]: Connected | |
| 2020-04-21/22:19:04.549 [tgt 172.21.27.190:6060:18]: Handshake command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:04.550 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-21/22:19:04.550 IO sequence number:36996506 Degraded IO sequence number:35791038 | |
| 2020-04-21/22:19:04.552 New data connection on fd 24 | |
| 2020-04-21/22:19:04.552 ERROR zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 io receiver already present | |
| 2020-04-21/22:19:04.553 ERROR [tgt 172.21.133.201:6060:20]: Error on connection for sock(20) | |
| 2020-04-21/22:19:04.553 [tgt 172.21.133.201:6060:20]: Closing the connection | |
| 2020-04-21/22:19:04.553 Data connection closed on fd: 24 | |
| 2020-04-21/22:19:04.554 ERROR connection closed for fd(18) | |
| 2020-04-21/22:19:04.554 [tgt 172.21.27.190:6060:18]: Closing the connection | |
| 2020-04-21/22:19:04.902 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-21/22:19:04.902 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-21/22:19:04.902 Data connection closed on fd: 19 | |
| 2020-04-21/22:19:06.546 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-21/22:19:06.546 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-21/22:19:06.546 Data connection closed on fd: 14 | |
| 2020-04-21/22:19:08.253 [tgt 172.21.27.190:6060:17]: Connected | |
| 2020-04-21/22:19:08.254 [tgt 172.21.133.201:6060:19]: Connected | |
| 2020-04-21/22:19:08.255 [tgt 172.21.27.190:6060:17]: Handshake command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:08.256 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-21/22:19:08.256 IO sequence number:36996506 Degraded IO sequence number:35791038 | |
| 2020-04-21/22:19:08.258 New data connection on fd 20 | |
| 2020-04-21/22:19:08.259 ERROR fail on unavailable snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-21/22:19:08.715 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:08.716 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:08.790 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:08.790 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:08.795 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 20 | |
| 2020-04-21/22:19:08.795 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 fd: 20 | |
| 2020-04-21/22:19:09.946 [tgt 172.21.133.201:6060:19]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:09.946 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-21/22:19:09.946 IO sequence number:70688177 Degraded IO sequence number:70688175 | |
| 2020-04-21/22:19:09.949 New data connection on fd 25 | |
| 2020-04-21/22:19:09.950 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-21/22:19:10.014 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:10.014 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:10.055 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:10.055 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:10.058 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 25 | |
| 2020-04-21/22:19:10.058 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 25 | |
| 2020-04-21/22:19:10.921 [tgt 172.21.27.190:6060:17]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:20.943 [tgt 172.21.27.190:6060:17]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:21.068 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:21.073 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-21/22:19:21.074 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:24] at 172.20.2.87:3233 helping in rebuild | |
| 2020-04-21/22:19:21.107 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-21/22:19:21.109 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-21/22:19:21.109 IO sequence number:70688177 Degraded IO sequence number:70688175 | |
| 2020-04-21/22:19:21.109 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:27] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-21/22:19:21.109 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(24) | |
| 2020-04-21/22:19:21.110 New rebuild connection on fd 28 | |
| 2020-04-21/22:19:21.111 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(28) | |
| 2020-04-21/22:19:21.111 Checkpointed IO_seq: 70688177, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:21.117 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:21.117 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:21.117 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:19:21.125 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:21.125 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:21.125 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:19:21.316 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507561 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:21.317 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507561 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:30.965 [tgt 172.21.27.190:6060:17]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:30.971 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-21/22:19:30.971 [pvc-7193bd1f-3457-11e9-816f-0050569876a2:29] at 172.20.2.87:3233 helping in rebuild | |
| 2020-04-21/22:19:31.002 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-21/22:19:31.004 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-21/22:19:31.004 IO sequence number:36996506 Degraded IO sequence number:36996508 | |
| 2020-04-21/22:19:31.004 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2:30] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-21/22:19:31.005 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 on fd(29) | |
| 2020-04-21/22:19:31.005 New rebuild connection on fd 31 | |
| 2020-04-21/22:19:31.006 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(31) | |
| 2020-04-21/22:19:31.006 Checkpointed IO_seq: 36996506, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:31.042 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:31.042 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:31.043 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-21/22:19:31.067 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:31.067 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:31.067 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:19:31.085 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:31.085 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:31.085 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:19:31.088 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:31.262 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap36996506.1587507571 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:31.262 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap36996506.1587507571 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:33.658 Checkpointed IO_seq: 36996506, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:36.096 Checkpointed IO_seq: 36996506, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:38.630 Checkpointed IO_seq: 36996506, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:40.984 [tgt 172.21.27.190:6060:17]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:41.101 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:41.934 Checkpointed IO_seq: 70688177, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:43.973 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-21/22:19:44.358 Checkpointed IO_seq: 36996506, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:48.174 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-21/22:19:48.174 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap36996506.1587507571 | |
| 2020-04-21/22:19:48.181 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-21/22:19:48.181 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-21/22:19:48.243 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:48.243 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(31) | |
| 2020-04-21/22:19:48.394 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:48.394 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:48.399 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-21/22:19:50.997 [tgt 172.21.27.190:6060:17]: Replica status command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:51.000 [tgt 172.21.27.190:6060:17]: Prepare for rebuild command for zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:51.002 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 has zvol_guid:6897280536073243281 | |
| 2020-04-21/22:19:51.002 IO sequence number:36996506 Degraded IO sequence number:36996508 | |
| 2020-04-21/22:19:51.007 New rebuild connection on fd 29 | |
| 2020-04-21/22:19:51.008 Rebuild scanner started on zvol pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(29) | |
| 2020-04-21/22:19:51.008 Checkpointed IO_seq: 36996506, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:51.014 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:51.014 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:51.014 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@pvc-7193bd1f-3457-11e9-816f-0050569876a2_snapshot-20190916050418-8e00_1568610270467233181 | |
| 2020-04-21/22:19:51.019 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:51.020 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:51.020 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:19:51.025 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:51.025 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:51.025 holding already available snapshot pvc-7193bd1f-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:19:51.115 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:19:51.152 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap36996506.1587507591 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:19:51.152 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap36996506.1587507591 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:19:53.016 Checkpointed IO_seq: 36996506, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:53.959 Checkpointed IO_seq: 70688177, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:55.431 Checkpointed IO_seq: 36996506, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:57.549 Checkpointed IO_seq: 36996506, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:19:59.321 Checkpointed IO_seq: 70688177, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:01.077 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-21/22:20:01.136 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:02.951 Checkpointed IO_seq: 36996506, Rebuild Req offset: 42949672960, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:03.216 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-21/22:20:03.216 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507561 | |
| 2020-04-21/22:20:03.217 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-21/22:20:03.217 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-21/22:20:03.287 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:03.287 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(28) | |
| 2020-04-21/22:20:03.290 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:03.290 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:03.292 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-21/22:20:08.062 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2@.io_snap36996506.1587507591 | |
| 2020-04-21/22:20:08.168 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:08.168 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-7193bd1f-3457-11e9-816f-0050569876a2 from sock(29) | |
| 2020-04-21/22:20:11.153 [tgt 172.21.133.201:6060:19]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:11.158 [tgt 172.21.133.201:6060:19]: Prepare for rebuild command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:11.164 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:11242408171391940993 | |
| 2020-04-21/22:20:11.164 IO sequence number:70688177 Degraded IO sequence number:70688175 | |
| 2020-04-21/22:20:11.169 New rebuild connection on fd 27 | |
| 2020-04-21/22:20:11.169 Rebuild scanner started on zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(27) | |
| 2020-04-21/22:20:11.170 Checkpointed IO_seq: 70688177, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:11.183 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:20:11.183 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:20:11.183 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-21/22:20:11.186 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:20:11.186 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-21/22:20:11.186 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-21/22:20:11.744 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507611 status change: DEGRADED -> DEGRADED | |
| 2020-04-21/22:20:11.744 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507611 rebuild status change: INIT -> INIT | |
| 2020-04-21/22:20:13.767 Checkpointed IO_seq: 70688177, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:15.773 Checkpointed IO_seq: 70688177, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:20.597 Checkpointed IO_seq: 70688177, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2) | |
| 2020-04-21/22:20:26.566 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507611 | |
| 2020-04-21/22:20:26.633 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 | |
| 2020-04-21/22:20:26.634 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(27) | |
| 2020-04-22/02:36:54.757 [tgt 172.21.133.201:6060:19]: Destroy snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:14.968 [tgt 172.21.253.64:6060:21]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:15.031 ERROR Failed to create cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup: 17 | |
| 2020-04-22/02:52:15.031 ERROR [tgt 172.21.253.64:6060:21]: Error reply with status 1 for OP 14 | |
| 2020-04-22/02:52:15.031 Connection closed by the peer for socket(22) | |
| 2020-04-22/02:52:15.031 Waiting for conn_closed(1) and ack_sender (-1)for fd(22) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:15.032 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 closed on fd: 22 | |
| 2020-04-22/02:52:15.034 ERROR connection closed for fd(21) | |
| 2020-04-22/02:52:15.034 [tgt 172.21.253.64:6060:21]: Closing the connection | |
| 2020-04-22/02:52:16.622 [tgt 172.21.24.254:6060:23]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:16.704 ERROR Failed to create cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup: 17 | |
| 2020-04-22/02:52:16.709 ERROR [tgt 172.21.24.254:6060:23]: Error reply with status 1 for OP 14 | |
| 2020-04-22/02:52:16.705 Connection closed by the peer for socket(8) | |
| 2020-04-22/02:52:16.713 Waiting for conn_closed(1) and ack_sender (-1)for fd(8) to be false for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:16.713 Data connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 closed on fd: 8 | |
| 2020-04-22/02:52:16.714 ERROR [tgt 172.21.24.254:6060:23]: Error on connection for sock(23) | |
| 2020-04-22/02:52:16.715 [tgt 172.21.24.254:6060:23]: Closing the connection | |
| 2020-04-22/02:52:16.715 [tgt 172.21.253.64:6060:14]: Connected | |
| 2020-04-22/02:52:16.725 [tgt 172.21.253.64:6060:14]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:16.726 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-22/02:52:16.726 IO sequence number:22406722 Degraded IO sequence number:22051226 | |
| 2020-04-22/02:52:16.733 New data connection on fd 21 | |
| 2020-04-22/02:52:16.734 ERROR zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 io receiver already present | |
| 2020-04-22/02:52:16.735 Data connection closed on fd: 21 | |
| 2020-04-22/02:52:16.750 ERROR connection closed for fd(14) | |
| 2020-04-22/02:52:16.750 [tgt 172.21.253.64:6060:14]: Closing the connection | |
| 2020-04-22/02:52:17.032 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-22/02:52:17.033 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-22/02:52:17.033 Data connection closed on fd: 22 | |
| 2020-04-22/02:52:18.713 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED | |
| 2020-04-22/02:52:18.714 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT | |
| 2020-04-22/02:52:18.714 Data connection closed on fd: 8 | |
| 2020-04-22/02:52:18.755 [tgt 172.21.24.254:6060:8]: Connected | |
| 2020-04-22/02:52:19.944 [tgt 172.21.24.254:6060:8]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:19.945 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-22/02:52:19.945 IO sequence number:41651800 Degraded IO sequence number:41651800 | |
| 2020-04-22/02:52:19.962 New data connection on fd 22 | |
| 2020-04-22/02:52:19.970 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap | |
| 2020-04-22/02:52:20.210 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:20.210 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:20.449 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:20.449 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:20.453 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22 | |
| 2020-04-22/02:52:20.453 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22 | |
| 2020-04-22/02:52:21.951 [tgt 172.21.253.64:6060:21]: Connected | |
| 2020-04-22/02:52:22.017 [tgt 172.21.253.64:6060:21]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:22.018 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-22/02:52:22.018 IO sequence number:22406722 Degraded IO sequence number:22051226 | |
| 2020-04-22/02:52:23.178 New data connection on fd 24 | |
| 2020-04-22/02:52:23.179 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap | |
| 2020-04-22/02:52:23.321 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:23.322 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:23.378 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:23.378 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:23.383 Started ack sender for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 24 | |
| 2020-04-22/02:52:23.383 Data connection associated with zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 24 | |
| 2020-04-22/02:52:32.626 [tgt 172.21.24.254:6060:8]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:34.396 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:34.402 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-22/02:52:34.402 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:26] at 172.20.15.240:3233 helping in rebuild | |
| 2020-04-22/02:52:34.459 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-22/02:52:34.460 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-22/02:52:34.460 IO sequence number:22406722 Degraded IO sequence number:22406727 | |
| 2020-04-22/02:52:34.461 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:27] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-22/02:52:34.461 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(26) | |
| 2020-04-22/02:52:34.462 New rebuild connection on fd 28 | |
| 2020-04-22/02:52:34.462 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(28) | |
| 2020-04-22/02:52:34.463 Checkpointed IO_seq: 22406722, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-22/02:52:34.475 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:34.475 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:34.475 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 | |
| 2020-04-22/02:52:34.490 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:34.490 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:34.490 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:34.539 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:34.539 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:36.610 Checkpointed IO_seq: 22406722, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-22/02:52:38.738 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-22/02:52:38.738 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954 | |
| 2020-04-22/02:52:38.881 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:38.882 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(28) | |
| 2020-04-22/02:52:39.089 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed | |
| 2020-04-22/02:52:39.090 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-22/02:52:39.090 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-22/02:52:39.245 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:39.245 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:39.247 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-22/02:52:42.646 [tgt 172.21.24.254:6060:8]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:42.672 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS | |
| 2020-04-22/02:52:42.673 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:23] at 172.20.12.180:3233 helping in rebuild | |
| 2020-04-22/02:52:42.736 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS | |
| 2020-04-22/02:52:42.737 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-22/02:52:42.737 IO sequence number:41651800 Degraded IO sequence number:41651823 | |
| 2020-04-22/02:52:42.737 [cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:27] at 172.20.11.37:3233 helping in rebuild | |
| 2020-04-22/02:52:42.738 Rebuild started from clone for vol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(23) | |
| 2020-04-22/02:52:42.743 New rebuild connection on fd 28 | |
| 2020-04-22/02:52:42.746 Rebuild scanner started on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(28) | |
| 2020-04-22/02:52:42.746 Checkpointed IO_seq: 41651800, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-22/02:52:42.748 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:42.748 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:42.749 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:42.755 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:42.755 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:42.755 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-22/02:52:42.930 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523962 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:42.930 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523962 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:44.417 [tgt 172.21.253.64:6060:21]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:44.419 [tgt 172.21.253.64:6060:21]: Prepare for rebuild command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:44.420 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:2332608751544743781 | |
| 2020-04-22/02:52:44.420 IO sequence number:22406722 Degraded IO sequence number:22406727 | |
| 2020-04-22/02:52:44.438 New rebuild connection on fd 26 | |
| 2020-04-22/02:52:44.439 Rebuild scanner started on zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26) | |
| 2020-04-22/02:52:44.439 Checkpointed IO_seq: 22406722, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-22/02:52:44.445 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:44.445 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:44.445 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 | |
| 2020-04-22/02:52:44.449 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:44.449 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:44.449 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:44.535 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-22/02:52:44.837 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap22406722.1587523964 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:44.837 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap22406722.1587523964 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:45.608 Rebuilding zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed | |
| 2020-04-22/02:52:45.608 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523962 | |
| 2020-04-22/02:52:45.609 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE | |
| 2020-04-22/02:52:45.609 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY | |
| 2020-04-22/02:52:45.690 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:45.690 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(28) | |
| 2020-04-22/02:52:45.691 Destroying cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:45.691 Closing cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:45.692 Destroy for pool: null vol: cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0 | |
| 2020-04-22/02:52:46.548 Checkpointed IO_seq: 22406722, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2) | |
| 2020-04-22/02:52:48.494 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@.io_snap22406722.1587523964 | |
| 2020-04-22/02:52:48.573 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 | |
| 2020-04-22/02:52:48.574 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26) | |
| 2020-04-22/02:52:52.688 [tgt 172.21.24.254:6060:8]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:52.697 [tgt 172.21.24.254:6060:8]: Prepare for rebuild command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:52.699 Volume:cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:16294957439751977030 | |
| 2020-04-22/02:52:52.699 IO sequence number:41651800 Degraded IO sequence number:41651823 | |
| 2020-04-22/02:52:52.707 New rebuild connection on fd 26 | |
| 2020-04-22/02:52:52.707 Rebuild scanner started on zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-22/02:52:52.708 Checkpointed IO_seq: 41651800, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2) | |
| 2020-04-22/02:52:52.710 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:52.710 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:52.710 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-22/02:52:52.715 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:52.715 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:52.715 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 | |
| 2020-04-22/02:52:52.805 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap41651800.1587523972 status change: DEGRADED -> DEGRADED | |
| 2020-04-22/02:52:52.805 zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap41651800.1587523972 rebuild status change: INIT -> INIT | |
| 2020-04-22/02:52:54.040 closing snap cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@.io_snap41651800.1587523972 | |
| 2020-04-22/02:52:54.131 Rebuild process is over on zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 | |
| 2020-04-22/02:52:54.132 Closing rebuild connection for zvol cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26) | |
| 2020-04-22/03:49:25.692 [tgt 172.21.24.254:6060:8]: Destroy snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup | |
| 2020-04-22/07:17:46.412 [tgt 172.21.133.201:6060:19]: Create snapshot command for cstor-026b3756-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-minio | |
| root@mlrack1:~# | |
| root@mlrack1:~# |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment