cstor-disk-gitlab-u4qx-67bdd7d787-nqvd2

gistfile1.txt

root@mlrack1:~# kubectl logs  cstor-disk-gitlab-u4qx-67bdd7d787-nqvd2 -n openebs -ccstor-pool
Disabling dumping core
sleeping for 2 sec
2020-04-15/13:22:25.794 disabled auto import (reading of zpool.cache)
physmem = 1048576 pages (4.00 GB)
2020-04-15/13:22:45.309 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.309 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> INIT
2020-04-15/13:22:45.310 Instantiating zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:22:45.314 [tgt 172.21.253.64:6060:18]: Connected
2020-04-15/13:22:45.315 [tgt 172.21.253.64:6060:18]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:22:45.315 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.315 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT
2020-04-15/13:22:45.316 Instantiating zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.320 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.320 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT
2020-04-15/13:22:45.321 Instantiating zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.347 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.348 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> INIT
2020-04-15/13:22:45.352 Instantiating zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.446 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-15/13:22:45.447 IO sequence number:21001690 Degraded IO sequence number:20929073
2020-04-15/13:22:45.449 [tgt 172.21.24.254:6060:14]: Connected
2020-04-15/13:22:45.449 [tgt 172.21.133.201:6060:8]: Connected
2020-04-15/13:22:45.450 [tgt 172.21.133.201:6060:8]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.452 New data connection on fd 20
2020-04-15/13:22:45.453 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-15/13:22:45.454 IO sequence number:69140511 Degraded IO sequence number:69057431
2020-04-15/13:22:45.454 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap
2020-04-15/13:22:45.454 [tgt 172.21.179.130:6060:19]: Connected
2020-04-15/13:22:45.454 [tgt 172.21.24.254:6060:14]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.455 New data connection on fd 21
2020-04-15/13:22:45.458 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:22:45.459 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-15/13:22:45.459 IO sequence number:40346880 Degraded IO sequence number:40294359
2020-04-15/13:22:45.459 [tgt 172.21.179.130:6060:19]: Handshake command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:22:45.462 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-15/13:22:45.462 IO sequence number:67134007 Degraded IO sequence number:66923086
2020-04-15/13:22:45.463 New data connection on fd 22
2020-04-15/13:22:45.465 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:22:45.466 New data connection on fd 23
2020-04-15/13:22:45.467 ERROR fail on unavailable snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:22:45.600 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.600 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:22:45.600 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.601 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:22:45.612 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.612 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:22:45.612 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.612 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:22:45.663 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.663 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:22:45.667 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 20
2020-04-15/13:22:45.667 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 20
2020-04-15/13:22:45.797 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.797 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:22:45.797 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.800 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:22:45.801 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 23
2020-04-15/13:22:45.804 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 21
2020-04-15/13:22:45.806 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:22:45.807 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:22:45.804 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 21
2020-04-15/13:22:45.811 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 23
2020-04-15/13:22:45.812 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22
2020-04-15/13:22:45.815 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22
2020-04-15/13:22:56.681 [tgt 172.21.253.64:6060:18]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:22:56.685 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-
fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:22:56.685 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:25] at 172.20.11.37:3233 helping in rebuild
2020-04-15/13:22:56.694 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:22:56.696 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-15/13:22:56.696 IO sequence number:21001690 Degraded IO sequence number:21002214
2020-04-15/13:22:56.696 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:26] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:22:56.697 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(25)
2020-04-15/13:22:56.697 New rebuild connection on fd 27
2020-04-15/13:22:56.698 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(27)
2020-04-15/13:22:56.698 Checkpointed IO_seq: 21001690, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:22:56.811 [tgt 172.21.179.130:6060:19]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:22:56.816 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:22:56.816 [pvc-6a48c495-3457-11e9-816f-0050569876a2:24] at 172.20.12.180:3233 helping in rebuild
2020-04-15/13:22:56.820 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001690.1586956976 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:56.820 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001690.1586956976 rebuild status change: INIT -> INIT
2020-04-15/13:22:56.821 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:22:56.821 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:22:56.823 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-15/13:22:56.823 IO sequence number:67134007 Degraded IO sequence number:67135797
2020-04-15/13:22:56.823 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2:28] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:22:56.824 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 on fd(24)
2020-04-15/13:22:56.825 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:22:56.825 New rebuild connection on fd 29
2020-04-15/13:22:56.826 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(29)
2020-04-15/13:22:56.826 Checkpointed IO_seq: 67134007, Rebuild Req offset: 0, Rebuild Req length: 8589934592 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2)
2020-04-15/13:22:56.827 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:22:56.827 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:30] at 172.20.11.37:3233 helping in rebuild
2020-04-15/13:22:56.831 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:22:56.832 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:22:56.832 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:31] at 172.20.11.37:3233 helping in rebuild
2020-04-15/13:22:56.833 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-15/13:22:56.834 IO sequence number:69140511 Degraded IO sequence number:69057431
2020-04-15/13:22:56.834 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:32] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:22:56.834 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(30)
2020-04-15/13:22:56.835 New rebuild connection on fd 33
2020-04-15/13:22:56.836 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(33)
2020-04-15/13:22:56.836 Checkpointed IO_seq: 69140511, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:22:56.846 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:22:56.847 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-15/13:22:56.847 IO sequence number:40346880 Degraded IO sequence number:40294359
2020-04-15/13:22:56.848 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:34] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:22:56.849 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(31)
2020-04-15/13:22:56.850 New rebuild connection on fd 35
2020-04-15/13:22:56.850 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(35)
2020-04-15/13:22:56.852 Checkpointed IO_seq: 40346880, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2)
2020-04-15/13:22:56.895 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586956976 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:56.896 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586956976 rebuild status change: INIT -> INIT
2020-04-15/13:22:57.055 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586956976 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:57.055 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586956976 rebuild status change: INIT -> INIT
2020-04-15/13:22:57.056 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586956976 status change: DEGRADED -> DEGRADED
2020-04-15/13:22:57.056 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586956976 rebuild status change: INIT -> INIT
2020-04-15/13:23:00.283 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:01.035 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:02.988 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-15/13:23:06.697 [tgt 172.21.253.64:6060:18]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:06.825 [tgt 172.21.179.130:6060:19]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:23:06.837 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:06.838 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:11.166 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:11.166 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586956976
2020-04-15/13:23:11.167 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:23:11.167 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:23:11.248 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:23:11.248 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(29)
2020-04-15/13:23:11.250 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:23:11.251 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:23:11.256 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:23:11.541 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:16.710 [tgt 172.21.253.64:6060:18]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:16.840 [tgt 172.21.179.130:6060:19]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:23:16.851 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:16.851 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:17.442 Checkpointed IO_seq: 21001690, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:23:19.324 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:19.325 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586956976
2020-04-15/13:23:19.325 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:23:19.325 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:23:19.514 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:19.515 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(35)
2020-04-15/13:23:19.519 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:19.520 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:19.521 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:23:21.187 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-15/13:23:21.187 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21001690.1586956976
2020-04-15/13:23:21.188 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:23:21.188 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:23:21.293 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:21.293 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(27)
2020-04-15/13:23:21.370 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:21.370 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:21.375 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:23:26.723 [tgt 172.21.253.64:6060:18]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:23:26.856 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:23:26.868 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:28.899 Checkpointed IO_seq: 69140511, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:23:36.881 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:40.337 Checkpointed IO_seq: 69140511, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:23:46.895 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:51.873 Checkpointed IO_seq: 69140511, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:23:55.838 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:23:55.838 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:23:55.839 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:23:55.839 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586956976
2020-04-15/13:23:55.987 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:55.988 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(33)
2020-04-15/13:23:55.988 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:55.988 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:23:55.995 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:23:56.903 [tgt 172.21.133.201:6060:8]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:29:47.395 ERROR connection closed for fd(8)
2020-04-15/13:29:47.395 [tgt 172.21.133.201:6060:8]: Closing the connection
2020-04-15/13:29:47.397 Connection closed by the peer for socket(21)
2020-04-15/13:29:47.397 Waiting for conn_closed(1) and ack_sender (-1)for fd(21) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:29:47.398 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 closed on fd: 21
2020-04-15/13:29:49.398 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-15/13:29:49.398 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-15/13:29:49.398 Data connection closed on fd: 21
2020-04-15/13:29:52.406 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:29:55.411 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:30:02.331 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:05.338 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:12.344 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:30:17.353 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:20.359 ERROR Failed to connect to 172.21.133.201:6060 fd(21)
2020-04-15/13:30:27.370 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:30:30.377 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:30:37.390 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:40.396 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:47.405 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:30:50.411 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:30:57.421 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:31:00.427 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:31:07.439 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:31:10.446 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:31:17.457 ERROR Failed to connect to 172.21.133.201:6060 fd(8)
2020-04-15/13:31:20.463 ERROR Failed to connect to 172.21.133.201:6060 fd(17)
2020-04-15/13:31:24.472 [tgt 172.21.133.201:6060:17]: Connected
2020-04-15/13:31:25.481 [tgt 172.21.133.201:6060:17]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:31:25.482 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-15/13:31:25.482 IO sequence number:69140511 Degraded IO sequence number:69143427
2020-04-15/13:31:25.494 New data connection on fd 8
2020-04-15/13:31:25.497 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:31:25.741 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:31:25.741 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:31:25.816 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:31:25.816 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:31:25.820 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 8
2020-04-15/13:31:25.820 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 8
2020-04-15/13:31:37.969 [tgt 172.21.133.201:6060:17]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:31:47.634 Connection closed by the peer for socket(22)
2020-04-15/13:31:47.634 ERROR connection closed for fd(14)
2020-04-15/13:31:47.635 [tgt 172.21.24.254:6060:14]: Closing the connection
2020-04-15/13:31:47.635 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 closed on fd: 22
2020-04-15/13:31:47.634 Waiting for conn_closed(1) and ack_sender (-1)for fd(22) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:31:47.985 [tgt 172.21.133.201:6060:17]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:31:47.991 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:31:47.991 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:21] at 172.20.11.37:3233 helping in rebuild
2020-04-15/13:31:47.996 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:31:47.997 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-15/13:31:47.997 IO sequence number:69140511 Degraded IO sequence number:70192004
2020-04-15/13:31:47.998 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:25] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:31:47.998 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(21)
2020-04-15/13:31:47.998 New rebuild connection on fd 26
2020-04-15/13:31:47.999 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-15/13:31:47.999 Checkpointed IO_seq: 69140511, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:31:48.078 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586957507 status change: DEGRADED -> DEGRADED
2020-04-15/13:31:48.078 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586957507 rebuild status change: INIT -> INIT
2020-04-15/13:31:49.635 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-15/13:31:49.635 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-15/13:31:49.636 Data connection closed on fd: 22
2020-04-15/13:31:50.187 Checkpointed IO_seq: 69140511, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:31:50.991 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:31:52.061 Checkpointed IO_seq: 69140511, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:31:56.001 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:31:56.599 Checkpointed IO_seq: 69140511, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:31:58.003 [tgt 172.21.133.201:6060:17]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:31:59.010 ERROR Failed to connect to 172.21.24.254:6060 fd(22)
2020-04-15/13:32:00.005 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:32:00.729 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:32:00.729 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:32:00.729 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:32:00.729 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap69140511.1586957507
2020-04-15/13:32:00.856 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:00.856 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-15/13:32:00.857 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:00.857 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:00.858 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:32:04.017 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:32:07.023 ERROR Failed to connect to 172.21.24.254:6060 fd(21)
2020-04-15/13:32:08.013 [tgt 172.21.133.201:6060:17]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:08.016 [tgt 172.21.133.201:6060:17]: Prepare for rebuild command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:08.016 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-15/13:32:08.017 IO sequence number:69140511 Degraded IO sequence number:70192004
2020-04-15/13:32:08.023 New rebuild connection on fd 21
2020-04-15/13:32:08.024 Rebuild scanner started on zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(21)
2020-04-15/13:32:08.024 Checkpointed IO_seq: 69141164, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:32:08.148 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69141164.1586957528 status change: DEGRADED -> DEGRADED
2020-04-15/13:32:08.148 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69141164.1586957528 rebuild status change: INIT -> INIT
2020-04-15/13:32:10.591 Checkpointed IO_seq: 69141164, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:32:12.464 Checkpointed IO_seq: 69141164, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:32:13.025 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:32:19.071 Checkpointed IO_seq: 69141164, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-15/13:32:21.044 ERROR Failed to connect to 172.21.24.254:6060 fd(22)
2020-04-15/13:32:23.438 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap69141164.1586957528
2020-04-15/13:32:23.561 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-15/13:32:23.561 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(21)
2020-04-15/13:32:23.593 ERROR connection closed for fd(19)
2020-04-15/13:32:23.593 Connection closed by the peer for socket(23)
2020-04-15/13:32:23.593 Waiting for conn_closed(1) and ack_sender (-1)for fd(23) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:32:23.593 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 closed on fd: 23
2020-04-15/13:32:23.593 [tgt 172.21.179.130:6060:19]: Closing the connection
2020-04-15/13:32:24.049 ERROR Failed to connect to 172.21.24.254:6060 fd(22)
2020-04-15/13:32:25.593 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-15/13:32:25.594 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-15/13:32:25.594 Data connection closed on fd: 23
2020-04-15/13:32:29.057 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:32:32.065 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:32:32.065 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:32:35.073 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:32:40.082 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:32:43.056 ERROR Failed to connect to 172.21.24.254:6060 fd(21)
2020-04-15/13:32:43.089 ERROR Failed to connect to 172.21.179.130:6060 fd(19)
2020-04-15/13:32:48.097 ERROR Failed to connect to 172.21.24.254:6060 fd(21)
2020-04-15/13:32:50.101 ERROR Failed to connect to 172.21.179.130:6060 fd(22)
2020-04-15/13:32:51.104 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:32:53.107 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:32:58.115 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:33:00.117 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:01.121 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:33:06.129 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:08.134 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:33:09.137 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:11.142 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:33:14.145 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:17.151 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:19.153 ERROR Failed to connect to 172.21.24.254:6060 fd(19)
2020-04-15/13:33:22.160 ERROR Failed to connect to 172.21.24.254:6060 fd(14)
2020-04-15/13:33:24.161 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:26.162 [tgt 172.21.24.254:6060:19]: Connected
2020-04-15/13:33:26.164 [tgt 172.21.24.254:6060:19]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:26.165 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-15/13:33:26.165 IO sequence number:40346880 Degraded IO sequence number:40347553
2020-04-15/13:33:26.169 New data connection on fd 22
2020-04-15/13:33:26.171 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:33:26.225 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:33:26.226 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:33:26.276 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:33:26.276 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:33:26.278 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22
2020-04-15/13:33:26.278 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 22
2020-04-15/13:33:27.167 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:27.281 [tgt 172.21.24.254:6060:19]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:33.111 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:33:38.121 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:41.128 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:33:41.769 [tgt 172.21.24.254:6060:19]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:41.772 [tgt 172.21.24.254:6060:19]: Prepare for rebuild command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:41.773 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-15/13:33:41.773 IO sequence number:40346880 Degraded IO sequence number:41396156
2020-04-15/13:33:41.777 New rebuild connection on fd 21
2020-04-15/13:33:41.777 Rebuild scanner started on zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(21)
2020-04-15/13:33:41.777 Checkpointed IO_seq: 40346593, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2)
2020-04-15/13:33:41.889 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621 status change: DEGRADED -> DEGRADED
2020-04-15/13:33:41.889 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621 rebuild status change: INIT -> INIT
2020-04-15/13:33:43.492 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346593.1586957621
2020-04-15/13:33:43.632 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:43.633 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(21)
2020-04-15/13:33:48.783 ERROR Failed to connect to 172.21.179.130:6060 fd(23)
2020-04-15/13:33:51.786 [tgt 172.21.24.254:6060:19]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:33:53.792 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:33:58.804 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:34:01.802 [tgt 172.21.24.254:6060:19]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:34:01.807 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:34:01.808 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:34:01.808 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:14] at 172.20.12.180:3233 helping in rebuild
2020-04-15/13:34:01.812 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:34:01.813 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-15/13:34:01.813 IO sequence number:40346880 Degraded IO sequence number:41396156
2020-04-15/13:34:01.813 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:21] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:34:01.814 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(14)
2020-04-15/13:34:01.816 New rebuild connection on fd 24
2020-04-15/13:34:01.816 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(24)
2020-04-15/13:34:01.816 Checkpointed IO_seq: 40346880, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2)
2020-04-15/13:34:01.872 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586957641 status change: DEGRADED -> DEGRADED
2020-04-15/13:34:01.872 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586957641 rebuild status change: INIT -> INIT
2020-04-15/13:34:03.406 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:34:03.454 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:34:03.455 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap40346880.1586957641
2020-04-15/13:34:03.455 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:34:03.455 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:34:03.910 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:34:03.910 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(24)
2020-04-15/13:34:03.913 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:34:03.914 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:34:03.916 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:34:08.817 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:34:11.816 [tgt 172.21.24.254:6060:19]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-15/13:34:11.823 ERROR Failed to connect to 172.21.179.130:6060 fd(21)
2020-04-15/13:34:18.833 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:34:21.839 ERROR Failed to connect to 172.21.179.130:6060 fd(14)
2020-04-15/13:34:25.849 [tgt 172.21.179.130:6060:21]: Connected
2020-04-15/13:34:25.852 [tgt 172.21.179.130:6060:21]: Handshake command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:25.852 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-15/13:34:25.853 IO sequence number:67134007 Degraded IO sequence number:67136572
2020-04-15/13:34:25.857 New data connection on fd 23
2020-04-15/13:34:25.859 ERROR fail on unavailable snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-15/13:34:26.185 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:34:26.185 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:34:26.233 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:34:26.233 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:34:26.235 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 23
2020-04-15/13:34:26.235 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 23
2020-04-15/13:34:38.381 [tgt 172.21.179.130:6060:21]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:48.400 [tgt 172.21.179.130:6060:21]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:48.407 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:34:48.407 [pvc-6a48c495-3457-11e9-816f-0050569876a2:25] at 172.20.2.87:3233 helping in rebuild
2020-04-15/13:34:48.412 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:34:48.413 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-15/13:34:48.413 IO sequence number:67134007 Degraded IO sequence number:67136572
2020-04-15/13:34:48.413 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2:26] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:34:48.413 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 on fd(25)
2020-04-15/13:34:48.414 New rebuild connection on fd 27
2020-04-15/13:34:48.414 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(27)
2020-04-15/13:34:48.414 Checkpointed IO_seq: 67134007, Rebuild Req offset: 0, Rebuild Req length: 8589934592 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2)
2020-04-15/13:34:48.476 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586957688 status change: DEGRADED -> DEGRADED
2020-04-15/13:34:48.476 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586957688 rebuild status change: INIT -> INIT
2020-04-15/13:34:50.683 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:34:50.683 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap67134007.1586957688
2020-04-15/13:34:50.795 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:50.795 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(27)
2020-04-15/13:34:51.284 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-15/13:34:51.285 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:34:51.285 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:34:51.401 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:51.401 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:51.402 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:34:53.643 ERROR connection closed for fd(18)
2020-04-15/13:34:53.644 [tgt 172.21.253.64:6060:18]: Closing the connection
2020-04-15/13:34:53.645 Connection closed by the peer for socket(20)
2020-04-15/13:34:53.645 Waiting for conn_closed(1) and ack_sender (-1)for fd(20) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:34:53.646 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 closed on fd: 20
2020-04-15/13:34:55.646 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-15/13:34:55.646 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: DONE -> INIT
2020-04-15/13:34:55.646 Data connection closed on fd: 20
2020-04-15/13:34:56.652 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:34:58.418 [tgt 172.21.179.130:6060:21]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:58.420 [tgt 172.21.179.130:6060:21]: Prepare for rebuild command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:34:58.421 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-15/13:34:58.421 IO sequence number:67134007 Degraded IO sequence number:67136572
2020-04-15/13:34:58.426 New rebuild connection on fd 18
2020-04-15/13:34:58.426 Rebuild scanner started on zvol pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(18)
2020-04-15/13:34:58.426 Checkpointed IO_seq: 67133958, Rebuild Req offset: 0, Rebuild Req length: 8589934592 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2)
2020-04-15/13:34:58.489 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap67133958.1586957698 status change: DEGRADED -> DEGRADED
2020-04-15/13:34:58.489 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap67133958.1586957698 rebuild status change: INIT -> INIT
2020-04-15/13:34:59.657 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:00.696 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap67133958.1586957698
2020-04-15/13:35:00.748 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-15/13:35:00.748 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(18)
2020-04-15/13:35:06.881 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:09.887 ERROR Failed to connect to 172.21.253.64:6060 fd(20)
2020-04-15/13:35:16.897 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:35:21.227 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:24.233 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:31.461 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:35:34.467 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:43.477 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:35:46.483 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:53.481 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:35:56.487 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:36:03.488 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:36:08.498 ERROR Failed to connect to 172.21.253.64:6060 fd(18)
2020-04-15/13:36:11.505 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:36:18.959 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:36:23.967 ERROR Failed to connect to 172.21.253.64:6060 fd(14)
2020-04-15/13:36:25.971 [tgt 172.21.253.64:6060:14]: Connected
2020-04-15/13:36:25.972 [tgt 172.21.253.64:6060:14]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:36:25.972 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-15/13:36:25.972 IO sequence number:21002229 Degraded IO sequence number:21002614
2020-04-15/13:36:25.974 New data connection on fd 20
2020-04-15/13:36:25.975 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap
2020-04-15/13:36:26.040 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-15/13:36:26.040 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-15/13:36:26.086 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-15/13:36:26.086 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-15/13:36:26.090 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 20
2020-04-15/13:36:26.090 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 20
2020-04-15/13:36:41.639 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:36:41.641 [tgt 172.21.253.64:6060:14]: Prepare for rebuild command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:36:41.642 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-15/13:36:41.642 IO sequence number:21002229 Degraded IO sequence number:22051207
2020-04-15/13:36:41.646 New rebuild connection on fd 24
2020-04-15/13:36:41.646 Rebuild scanner started on zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(24)
2020-04-15/13:36:41.646 Checkpointed IO_seq: 21002085, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:36:41.700 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002085.1586957801 status change: DEGRADED -> DEGRADED
2020-04-15/13:36:41.700 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002085.1586957801 rebuild status change: INIT -> INIT
2020-04-15/13:36:44.121 Checkpointed IO_seq: 21002085, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:36:46.868 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002085.1586957801
2020-04-15/13:36:46.925 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:36:46.925 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(24)
2020-04-15/13:36:51.649 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:01.660 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:11.669 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:21.682 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:31.695 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:31.701 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-15/13:37:31.701 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:24] at 172.20.11.37:3233 helping in rebuild
2020-04-15/13:37:31.707 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-15/13:37:31.708 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-15/13:37:31.708 IO sequence number:21002229 Degraded IO sequence number:22051236
2020-04-15/13:37:31.708 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:25] at 172.20.15.240:3233 helping in rebuild
2020-04-15/13:37:31.709 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(24)
2020-04-15/13:37:31.709 New rebuild connection on fd 26
2020-04-15/13:37:31.710 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26)
2020-04-15/13:37:31.710 Checkpointed IO_seq: 21002229, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:37:31.774 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002229.1586957851 status change: DEGRADED -> DEGRADED
2020-04-15/13:37:31.774 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002229.1586957851 rebuild status change: INIT -> INIT
2020-04-15/13:37:34.457 Checkpointed IO_seq: 21002229, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-15/13:37:34.496 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-15/13:37:36.338 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-15/13:37:36.338 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap21002229.1586957851
2020-04-15/13:37:36.339 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-15/13:37:36.339 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-15/13:37:36.518 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:36.519 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(26)
2020-04-15/13:37:36.524 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:36.525 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/13:37:36.527 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0
2020-04-15/13:37:41.707 [tgt 172.21.253.64:6060:14]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-15/21:00:35.445 ERROR Pool cstor-023de05a-344a-11e9-816f-0050569876a2 took more than 15 seconds to sync
2020-04-21/22:00:21.539 [tgt 172.21.133.201:6060:17]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:00:23.578 [tgt 172.21.179.130:6060:21]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:00:25.614 [tgt 172.21.253.64:6060:14]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup
2020-04-21/22:00:27.684 [tgt 172.21.24.254:6060:19]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:09:21.389 [tgt 172.21.133.201:6060:17]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-21/22:09:23.361 [tgt 172.21.179.130:6060:21]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-21/22:09:25.330 [tgt 172.21.253.64:6060:14]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1
2020-04-21/22:09:27.369 [tgt 172.21.24.254:6060:19]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-21/22:19:04.512 [tgt 172.21.133.201:6060:17]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:19:04.599 ERROR Failed to create cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup: 17
2020-04-21/22:19:04.599 Connection closed by the peer for socket(8)
2020-04-21/22:19:04.599 Waiting for conn_closed(1) and ack_sender (-1)for fd(8) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:19:04.599 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 closed on fd: 8
2020-04-21/22:19:04.599 ERROR [tgt 172.21.133.201:6060:17]: Error reply with status 1 for OP 14
2020-04-21/22:19:04.601 ERROR [tgt 172.21.133.201:6060:17]: Error on connection for sock(17)
2020-04-21/22:19:04.601 [tgt 172.21.133.201:6060:17]: Closing the connection
2020-04-21/22:19:06.600 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-21/22:19:06.600 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-21/22:19:06.600 Data connection closed on fd: 8
2020-04-21/22:19:08.259 [tgt 172.21.133.201:6060:18]: Connected
2020-04-21/22:19:08.261 [tgt 172.21.133.201:6060:18]: Handshake command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:19:08.262 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-21/22:19:08.262 IO sequence number:70688177 Degraded IO sequence number:70688175
2020-04-21/22:19:08.267 New data connection on fd 24
2020-04-21/22:19:08.274 ERROR fail on unavailable snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-21/22:19:08.880 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-21/22:19:08.880 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-21/22:19:08.945 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-21/22:19:08.945 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-21/22:19:08.949 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 24
2020-04-21/22:19:08.949 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 fd: 24
2020-04-21/22:19:21.073 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:19:31.091 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:19:41.106 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:19:51.120 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:01.134 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:11.158 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:11.172 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-21/22:20:11.172 [pvc-6a4665f5-3457-11e9-816f-0050569876a2:17] at 172.20.11.37:3233 helping in rebuild
2020-04-21/22:20:11.194 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-21/22:20:11.195 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-21/22:20:11.195 IO sequence number:70688177 Degraded IO sequence number:70688175
2020-04-21/22:20:11.195 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2:26] at 172.20.15.240:3233 helping in rebuild
2020-04-21/22:20:11.195 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 on fd(17)
2020-04-21/22:20:11.196 New rebuild connection on fd 27
2020-04-21/22:20:11.196 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(27)
2020-04-21/22:20:11.196 Checkpointed IO_seq: 70688177, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:11.209 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-21/22:20:11.209 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-21/22:20:11.209 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:20:11.226 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-21/22:20:11.226 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-21/22:20:11.226 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-21/22:20:11.616 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507611 status change: DEGRADED -> DEGRADED
2020-04-21/22:20:11.617 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507611 rebuild status change: INIT -> INIT
2020-04-21/22:20:21.185 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:26.571 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-21/22:20:28.144 Checkpointed IO_seq: 70688177, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:31.203 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:41.232 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:42.443 Checkpointed IO_seq: 70688177, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:47.460 Checkpointed IO_seq: 70688177, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:50.763 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 completed
2020-04-21/22:20:50.764 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap70688177.1587507611
2020-04-21/22:20:50.764 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-21/22:20:50.764 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-21/22:20:51.040 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:51.040 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(27)
2020-04-21/22:20:51.246 [tgt 172.21.133.201:6060:18]: Replica status command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:51.250 [tgt 172.21.133.201:6060:18]: Prepare for rebuild command for zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:51.263 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 has zvol_guid:9938091502532204351
2020-04-21/22:20:51.264 IO sequence number:70688177 Degraded IO sequence number:70688175
2020-04-21/22:20:51.275 New rebuild connection on fd 17
2020-04-21/22:20:51.278 Rebuild scanner started on zvol pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-21/22:20:51.278 Checkpointed IO_seq: 70688177, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:51.283 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-21/22:20:51.283 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-21/22:20:51.283 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-21/22:20:51.312 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-21/22:20:51.312 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-21/22:20:51.312 holding already available snapshot pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-21/22:20:51.510 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:51.510 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:20:51.511 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-21/22:20:51.521 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507651 status change: DEGRADED -> DEGRADED
2020-04-21/22:20:51.521 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507651 rebuild status change: INIT -> INIT
2020-04-21/22:20:53.289 Checkpointed IO_seq: 70688177, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:54.738 Checkpointed IO_seq: 70688177, Rebuild Req offset: 21474836480, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:20:57.326 Checkpointed IO_seq: 70688177, Rebuild Req offset: 32212254720, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2)
2020-04-21/22:21:00.283 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@.io_snap70688177.1587507651
2020-04-21/22:21:00.390 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2
2020-04-21/22:21:00.390 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-22/02:36:54.762 [tgt 172.21.133.201:6060:18]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:37:00.400 [tgt 172.21.179.130:6060:21]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:37:00.453 ERROR Failed to create cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup: 17
2020-04-22/02:37:00.454 Connection closed by the peer for socket(23)
2020-04-22/02:37:00.454 Waiting for conn_closed(1) and ack_sender (-1)for fd(23) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:00.454 ERROR [tgt 172.21.179.130:6060:21]: Error reply with status 1 for OP 14
2020-04-22/02:37:00.454 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 closed on fd: 23
2020-04-22/02:37:00.456 ERROR connection closed for fd(21)
2020-04-22/02:37:00.456 [tgt 172.21.179.130:6060:21]: Closing the connection
2020-04-22/02:37:02.454 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-22/02:37:02.455 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-22/02:37:02.455 Data connection closed on fd: 23
2020-04-22/02:37:02.463 [tgt 172.21.179.130:6060:8]: Connected
2020-04-22/02:37:03.717 [tgt 172.21.179.130:6060:8]: Handshake command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:03.718 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-22/02:37:03.718 IO sequence number:68709913 Degraded IO sequence number:67136572
2020-04-22/02:37:05.177 New data connection on fd 21
2020-04-22/02:37:05.178 ERROR fail on unavailable snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-22/02:37:05.219 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-22/02:37:05.219 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-22/02:37:05.257 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-22/02:37:05.257 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-22/02:37:05.260 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 21
2020-04-22/02:37:05.260 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 fd: 21
2020-04-22/02:37:06.270 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-22/02:37:06.271 [pvc-6a48c495-3457-11e9-816f-0050569876a2:17] at 172.20.2.87:3233 helping in rebuild
2020-04-22/02:37:06.382 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-22/02:37:06.383 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-22/02:37:06.384 IO sequence number:68709913 Degraded IO sequence number:67136572
2020-04-22/02:37:06.384 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2:23] at 172.20.15.240:3233 helping in rebuild
2020-04-22/02:37:06.384 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 on fd(17)
2020-04-22/02:37:06.386 New rebuild connection on fd 26
2020-04-22/02:37:06.386 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-22/02:37:06.386 Checkpointed IO_seq: 68709913, Rebuild Req offset: 0, Rebuild Req length: 8589934592 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2)
2020-04-22/02:37:06.407 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:37:06.407 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:37:06.407 holding already available snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:37:06.425 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:37:06.425 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:37:06.425 holding already available snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-22/02:37:06.515 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap68709913.1587523026 status change: DEGRADED -> DEGRADED
2020-04-22/02:37:06.516 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap68709913.1587523026 rebuild status change: INIT -> INIT
2020-04-22/02:37:11.217 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-22/02:37:12.477 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 completed
2020-04-22/02:37:12.478 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-22/02:37:12.478 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-22/02:37:12.483 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap68709913.1587523026
2020-04-22/02:37:12.607 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:12.607 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-22/02:37:12.615 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:12.615 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:12.619 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-22/02:37:16.282 [tgt 172.21.179.130:6060:8]: Replica status command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:16.285 [tgt 172.21.179.130:6060:8]: Prepare for rebuild command for zvol pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:16.286 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 has zvol_guid:15014498555980183447
2020-04-22/02:37:16.286 IO sequence number:68709913 Degraded IO sequence number:68709926
2020-04-22/02:37:16.292 New rebuild connection on fd 17
2020-04-22/02:37:16.292 Rebuild scanner started on zvol pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-22/02:37:16.292 Checkpointed IO_seq: 68709913, Rebuild Req offset: 0, Rebuild Req length: 8589934592 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2)
2020-04-22/02:37:16.294 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:37:16.294 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:37:16.294 holding already available snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:37:16.296 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:37:16.296 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:37:16.296 holding already available snapshot pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-22/02:37:16.351 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap68709913.1587523036 status change: DEGRADED -> DEGRADED
2020-04-22/02:37:16.352 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap68709913.1587523036 rebuild status change: INIT -> INIT
2020-04-22/02:37:19.362 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@.io_snap68709913.1587523036
2020-04-22/02:37:19.540 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2
2020-04-22/02:37:19.540 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-22/02:52:13.051 [tgt 172.21.179.130:6060:8]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:52:14.972 [tgt 172.21.253.64:6060:14]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup
2020-04-22/02:52:15.056 ERROR Failed to create cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup: 17
2020-04-22/02:52:15.056 Connection closed by the peer for socket(20)
2020-04-22/02:52:15.057 Waiting for conn_closed(1) and ack_sender (-1)for fd(20) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:15.057 ERROR [tgt 172.21.253.64:6060:14]: Error reply with status 1 for OP 14
2020-04-22/02:52:15.057 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 closed on fd: 20
2020-04-22/02:52:15.060 ERROR connection closed for fd(14)
2020-04-22/02:52:15.060 [tgt 172.21.253.64:6060:14]: Closing the connection
2020-04-22/02:52:16.627 [tgt 172.21.24.254:6060:19]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:52:16.710 ERROR Failed to create cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup: 17
2020-04-22/02:52:16.710 Connection closed by the peer for socket(22)
2020-04-22/02:52:16.710 Waiting for conn_closed(1) and ack_sender (-1)for fd(22) to be false for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:16.710 ERROR [tgt 172.21.24.254:6060:19]: Error reply with status 1 for OP 14
2020-04-22/02:52:16.710 Data connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 closed on fd: 22
2020-04-22/02:52:16.719 [tgt 172.21.253.64:6060:17]: Connected
2020-04-22/02:52:16.719 [tgt 172.21.253.64:6060:17]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:16.719 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-22/02:52:16.719 IO sequence number:22406722 Degraded IO sequence number:22051238
2020-04-22/02:52:16.722 New data connection on fd 23
2020-04-22/02:52:16.722 ERROR [tgt 172.21.24.254:6060:19]: Error on connection for sock(19)
2020-04-22/02:52:16.722 [tgt 172.21.24.254:6060:19]: Closing the connection
2020-04-22/02:52:16.723 ERROR zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 io receiver already present
2020-04-22/02:52:16.724 Data connection closed on fd: 23
2020-04-22/02:52:16.726 ERROR connection closed for fd(17)
2020-04-22/02:52:16.726 [tgt 172.21.253.64:6060:17]: Closing the connection
2020-04-22/02:52:17.057 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-22/02:52:17.057 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: DONE -> INIT
2020-04-22/02:52:17.057 Data connection closed on fd: 20
2020-04-22/02:52:18.712 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: HEALTHY -> DEGRADED
2020-04-22/02:52:18.712 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: DONE -> INIT
2020-04-22/02:52:18.712 Data connection closed on fd: 22
2020-04-22/02:52:18.732 [tgt 172.21.24.254:6060:14]: Connected
2020-04-22/02:52:18.733 [tgt 172.21.24.254:6060:14]: Handshake command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:18.734 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-22/02:52:18.734 IO sequence number:41651800 Degraded IO sequence number:41396156
2020-04-22/02:52:18.737 New data connection on fd 19
2020-04-22/02:52:18.738 ERROR fail on unavailable snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap
2020-04-22/02:52:18.839 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-22/02:52:18.839 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-22/02:52:18.939 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-22/02:52:18.939 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-22/02:52:18.942 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 19
2020-04-22/02:52:18.942 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 fd: 19
2020-04-22/02:52:20.738 [tgt 172.21.253.64:6060:20]: Connected
2020-04-22/02:52:20.740 [tgt 172.21.253.64:6060:20]: Handshake command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:20.740 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-22/02:52:20.740 IO sequence number:22406722 Degraded IO sequence number:22051238
2020-04-22/02:52:20.742 New data connection on fd 22
2020-04-22/02:52:20.744 ERROR fail on unavailable snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap
2020-04-22/02:52:20.907 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap status change: DEGRADED -> DEGRADED
2020-04-22/02:52:20.907 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap rebuild status change: INIT -> INIT
2020-04-22/02:52:21.007 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone status change: DEGRADED -> DEGRADED
2020-04-22/02:52:21.007 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone rebuild status change: INIT -> INIT
2020-04-22/02:52:21.013 Started ack sender for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 22
2020-04-22/02:52:21.013 Data connection associated with zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 fd: 22
2020-04-22/02:52:32.632 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:32.635 [tgt 172.21.24.254:6060:14]: Prepare for rebuild command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:32.636 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-22/02:52:32.636 IO sequence number:41651800 Degraded IO sequence number:41651819
2020-04-22/02:52:32.641 New rebuild connection on fd 17
2020-04-22/02:52:32.642 Rebuild scanner started on zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-22/02:52:32.642 Checkpointed IO_seq: 41651800, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2)
2020-04-22/02:52:32.658 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:52:32.658 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:52:32.658 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:52:32.679 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:32.679 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:52:32.679 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-22/02:52:32.821 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523952 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:32.821 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523952 rebuild status change: INIT -> INIT
2020-04-22/02:52:34.400 [tgt 172.21.253.64:6060:20]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:34.402 [tgt 172.21.253.64:6060:20]: Prepare for rebuild command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:34.403 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-22/02:52:34.404 IO sequence number:22406722 Degraded IO sequence number:22406734
2020-04-22/02:52:34.410 New rebuild connection on fd 27
2020-04-22/02:52:34.411 Rebuild scanner started on zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(27)
2020-04-22/02:52:34.411 Checkpointed IO_seq: 22406722, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-22/02:52:34.422 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:52:34.422 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:52:34.422 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup
2020-04-22/02:52:34.459 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:34.459 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:52:34.459 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1
2020-04-22/02:52:34.546 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:34.546 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954 rebuild status change: INIT -> INIT
2020-04-22/02:52:34.669 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523952
2020-04-22/02:52:34.746 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:34.746 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(17)
2020-04-22/02:52:36.492 Checkpointed IO_seq: 22406722, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-22/02:52:39.095 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523954
2020-04-22/02:52:39.214 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:39.214 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(27)
2020-04-22/02:52:42.656 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:44.421 [tgt 172.21.253.64:6060:20]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:52.693 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:52.706 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-22/02:52:52.706 [pvc-6a4e7e60-3457-11e9-816f-0050569876a2:23] at 172.20.11.37:3233 helping in rebuild
2020-04-22/02:52:52.724 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-22/02:52:52.725 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 has zvol_guid:3668266241842055001
2020-04-22/02:52:52.725 IO sequence number:41651800 Degraded IO sequence number:41651823
2020-04-22/02:52:52.725 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2:25] at 172.20.15.240:3233 helping in rebuild
2020-04-22/02:52:52.726 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 on fd(23)
2020-04-22/02:52:52.726 New rebuild connection on fd 26
2020-04-22/02:52:52.729 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-22/02:52:52.729 Checkpointed IO_seq: 41651800, Rebuild Req offset: 0, Rebuild Req length: 5368709120 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2)
2020-04-22/02:52:52.734 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:52:52.734 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:52:52.734 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/02:52:52.740 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:52.740 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:52:52.741 holding already available snapshot pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-1
2020-04-22/02:52:52.825 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523972 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:52.825 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523972 rebuild status change: INIT -> INIT
2020-04-22/02:52:54.044 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-22/02:52:54.122 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 completed
2020-04-22/02:52:54.122 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone@.io_snap41651800.1587523972
2020-04-22/02:52:54.122 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-22/02:52:54.122 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-22/02:52:54.450 [tgt 172.21.253.64:6060:20]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:54.461 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: INIT -> SNAP REBUILD INPROGRESS
2020-04-22/02:52:54.462 [pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:17] at 172.20.12.180:3233 helping in rebuild
2020-04-22/02:52:54.475 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: SNAP REBUILD INPROGRESS -> ACTIVE DATASET REBUILD INPROGRESS
2020-04-22/02:52:54.476 Volume:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 has zvol_guid:9737481031741255553
2020-04-22/02:52:54.476 IO sequence number:22406722 Degraded IO sequence number:22406743
2020-04-22/02:52:54.476 [cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2:23] at 172.20.15.240:3233 helping in rebuild
2020-04-22/02:52:54.476 Rebuild started from clone for vol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 on fd(17)
2020-04-22/02:52:54.477 New rebuild connection on fd 29
2020-04-22/02:52:54.478 Rebuild scanner started on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(29)
2020-04-22/02:52:54.478 Checkpointed IO_seq: 22406722, Rebuild Req offset: 0, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-22/02:52:54.482 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup status change: DEGRADED -> DEGRADED
2020-04-22/02:52:54.482 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup rebuild status change: INIT -> INIT
2020-04-22/02:52:54.482 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup
2020-04-22/02:52:54.489 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:54.489 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1 rebuild status change: INIT -> INIT
2020-04-22/02:52:54.489 holding already available snapshot pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-1
2020-04-22/02:52:54.529 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:54.529 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2 from sock(26)
2020-04-22/02:52:54.624 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523974 status change: DEGRADED -> DEGRADED
2020-04-22/02:52:54.624 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523974 rebuild status change: INIT -> INIT
2020-04-22/02:52:54.627 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:54.627 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:52:54.628 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2_rebuild_clone, destroyed: 0
2020-04-22/02:52:55.851 Checkpointed IO_seq: 22406722, Rebuild Req offset: 10737418240, Rebuild Req length: 10737418240 for zvol(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2)
2020-04-22/02:52:56.990 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-22/02:52:56.991 closing snap cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone@.io_snap22406722.1587523974
2020-04-22/02:52:57.135 Rebuild process is over on zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:57.135 Closing rebuild connection for zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 from sock(29)
2020-04-22/02:52:58.254 Rebuilding zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 completed
2020-04-22/02:52:58.255 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 rebuild status change: ACTIVE DATASET REBUILD INPROGRESS -> DONE
2020-04-22/02:52:58.255 zvol cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2 status change: DEGRADED -> HEALTHY
2020-04-22/02:52:58.328 Destroying cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone(cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone) on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:58.328 Closing cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@rebuild_snap and cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone dataset on:cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/02:52:58.330 Destroy for pool: null vol: cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2_rebuild_clone, destroyed: 0
2020-04-22/02:53:02.721 [tgt 172.21.24.254:6060:14]: Replica status command for zvol pvc-6a4e7e60-3457-11e9-816f-0050569876a2
2020-04-22/02:53:04.474 [tgt 172.21.253.64:6060:20]: Replica status command for zvol pvc-e069fd37-fa38-11e9-b8bb-0050569876a2
2020-04-22/03:49:25.698 [tgt 172.21.24.254:6060:14]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup
2020-04-22/07:17:46.421 [tgt 172.21.133.201:6060:18]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-minio
2020-04-22/11:56:52.295 [tgt 172.21.133.201:6060:18]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4665f5-3457-11e9-816f-0050569876a2@gitlab-backup-minio
2020-04-22/12:07:38.454 [tgt 172.21.179.130:6060:8]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-postgresql
2020-04-22/12:22:24.643 [tgt 172.21.179.130:6060:8]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a48c495-3457-11e9-816f-0050569876a2@gitlab-backup-postgresql
2020-04-22/12:45:41.340 [tgt 172.21.253.64:6060:20]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-prometheus
2020-04-22/16:46:35.889 [tgt 172.21.253.64:6060:20]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-e069fd37-fa38-11e9-b8bb-0050569876a2@gitlab-backup-prometheus
2020-04-22/16:51:20.472 [tgt 172.21.24.254:6060:14]: Create snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-redis
2020-04-22/17:47:36.777 [tgt 172.21.24.254:6060:14]: Destroy snapshot command for cstor-023de05a-344a-11e9-816f-0050569876a2/pvc-6a4e7e60-3457-11e9-816f-0050569876a2@gitlab-backup-redis
root@mlrack1:~# 12:47