gistfile1.txt

[2014-08-12 19:19:23.518594] I [glusterfsd.c:1910:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.4.5 (/usr/sbin/glusterfs --volfile-id=/t0 --volfile-server=gluster1 /mnt)
[2014-08-12 19:19:23.520758] I [socket.c:3480:socket_init] 0-glusterfs: SSL support is NOT enabled
[2014-08-12 19:19:23.520780] I [socket.c:3495:socket_init] 0-glusterfs: using system polling thread
[2014-08-12 19:19:23.529828] I [socket.c:3480:socket_init] 0-t0-client-1: SSL support is NOT enabled
[2014-08-12 19:19:23.529842] I [socket.c:3495:socket_init] 0-t0-client-1: using system polling thread
[2014-08-12 19:19:23.530272] I [socket.c:3480:socket_init] 0-t0-client-0: SSL support is NOT enabled
[2014-08-12 19:19:23.530281] I [socket.c:3495:socket_init] 0-t0-client-0: using system polling thread
[2014-08-12 19:19:23.530298] I [client.c:2155:notify] 0-t0-client-0: parent translators are ready, attempting connect on transport
[2014-08-12 19:19:23.532879] I [client.c:2155:notify] 0-t0-client-1: parent translators are ready, attempting connect on transport
Given volfile:
+------------------------------------------------------------------------------+
  1: volume t0-client-0
  2:     type protocol/client
  3:     option transport-type tcp
  4:     option remote-subvolume /bricks/brick1/brick
  5:     option remote-host gluster1
  6: end-volume
  7:
  8: volume t0-client-1
  9:     type protocol/client
 10:     option transport-type tcp
 11:     option remote-subvolume /bricks/brick1/brick
 12:     option remote-host gluster2
 13: end-volume
 14:
 15: volume t0-replicate-0
 16:     type cluster/replicate
 17:     subvolumes t0-client-0 t0-client-1
 18: end-volume
 19:
 20: volume t0-dht
 21:     type cluster/distribute
 22:     subvolumes t0-replicate-0
 23: end-volume
 24:
 25: volume t0-write-behind
 26:     type performance/write-behind
 27:     subvolumes t0-dht
 28: end-volume
 29:
 30: volume t0-read-ahead
 31:     type performance/read-ahead
 32:     subvolumes t0-write-behind
 33: end-volume
 34:
 35: volume t0-io-cache
 36:     type performance/io-cache
 37:     subvolumes t0-read-ahead
 38: end-volume
 39:
 40: volume t0-quick-read
 41:     type performance/quick-read
 42:     subvolumes t0-io-cache
 43: end-volume
 44:
 45: volume t0-open-behind
 46:     type performance/open-behind
 47:     subvolumes t0-quick-read
 48: end-volume
 49:
 50: volume t0-md-cache
 51:     type performance/md-cache
 52:     subvolumes t0-open-behind
 53: end-volume
 54:
 55: volume t0
 56:     type debug/io-stats
 57:     option count-fop-hits off
 58:     option latency-measurement off
 59:     subvolumes t0-md-cache
 60: end-volume

+------------------------------------------------------------------------------+
[2014-08-12 19:19:23.536195] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-0: changing port to 49152 (from 0)
[2014-08-12 19:19:23.536239] W [socket.c:514:__socket_rwv] 0-t0-client-0: readv failed (No data available)
[2014-08-12 19:19:23.538753] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-1: changing port to 49152 (from 0)
[2014-08-12 19:19:23.538796] W [socket.c:514:__socket_rwv] 0-t0-client-1: readv failed (No data available)
[2014-08-12 19:19:23.541552] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:19:23.541642] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:19:23.541946] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-1: Connected to 192.168.30.108:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:19:23.541957] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:19:23.542033] I [afr-common.c:3698:afr_notify] 0-t0-replicate-0: Subvolume 't0-client-1' came back up; going online.
[2014-08-12 19:19:23.542161] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-1: Server lk version = 1
[2014-08-12 19:19:23.542286] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-0: Connected to 192.168.30.107:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:19:23.542296] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:19:23.544983] I [fuse-bridge.c:4771:fuse_graph_setup] 0-fuse: switched to graph 0
[2014-08-12 19:19:23.545031] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-0: Server lk version = 1
[2014-08-12 19:19:23.545178] I [fuse-bridge.c:3726:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.13 kernel 7.13
[2014-08-12 19:19:23.545737] I [afr-common.c:2057:afr_set_root_inode_on_first_lookup] 0-t0-replicate-0: added root inode
[2014-08-12 19:20:50.795226] I [glusterfsd-mgmt.c:58:mgmt_cbk_spec] 0-mgmt: Volume file changed
[2014-08-12 19:20:50.806449] I [socket.c:3480:socket_init] 1-t0-client-3: SSL support is NOT enabled
[2014-08-12 19:20:50.806463] I [socket.c:3495:socket_init] 1-t0-client-3: using system polling thread
[2014-08-12 19:20:50.806856] I [socket.c:3480:socket_init] 1-t0-client-2: SSL support is NOT enabled
[2014-08-12 19:20:50.806864] I [socket.c:3495:socket_init] 1-t0-client-2: using system polling thread
[2014-08-12 19:20:50.807263] I [socket.c:3480:socket_init] 1-t0-client-1: SSL support is NOT enabled
[2014-08-12 19:20:50.807273] I [socket.c:3495:socket_init] 1-t0-client-1: using system polling thread
[2014-08-12 19:20:50.807650] I [socket.c:3480:socket_init] 1-t0-client-0: SSL support is NOT enabled
[2014-08-12 19:20:50.807658] I [socket.c:3495:socket_init] 1-t0-client-0: using system polling thread
[2014-08-12 19:20:50.807669] I [client.c:2155:notify] 1-t0-client-0: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:50.810284] I [client.c:2155:notify] 1-t0-client-1: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:50.814981] I [client.c:2155:notify] 1-t0-client-2: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:50.819983] I [client.c:2155:notify] 1-t0-client-3: parent translators are ready, attempting connect on transport
Given volfile:
+------------------------------------------------------------------------------+
  1: volume t0-client-0
  2:     type protocol/client
  3:     option transport-type tcp
  4:     option remote-subvolume /bricks/brick1/brick
  5:     option remote-host gluster1
  6: end-volume
  7:
  8: volume t0-client-1
  9:     type protocol/client
 10:     option transport-type tcp
 11:     option remote-subvolume /bricks/brick1/brick
 12:     option remote-host gluster2
 13: end-volume
 14:
 15: volume t0-client-2
 16:     type protocol/client
 17:     option transport-type tcp
 18:     option remote-subvolume /bricks/brick2/brick
 19:     option remote-host gluster1
 20: end-volume
 21:
 22: volume t0-client-3
 23:     type protocol/client
 24:     option transport-type tcp
 25:     option remote-subvolume /bricks/brick2/brick
 26:     option remote-host gluster2
 27: end-volume
 28:
 29: volume t0-replicate-0
 30:     type cluster/replicate
 31:     subvolumes t0-client-0 t0-client-1
 32: end-volume
 33:
 34: volume t0-replicate-1
 35:     type cluster/replicate
 36:     subvolumes t0-client-2 t0-client-3
 37: end-volume
 38:
 39: volume t0-dht
 40:     type cluster/distribute
 41:     subvolumes t0-replicate-0 t0-replicate-1
 42: end-volume
 43:
 44: volume t0-write-behind
 45:     type performance/write-behind
 46:     subvolumes t0-dht
 47: end-volume
 48:
 49: volume t0-read-ahead
 50:     type performance/read-ahead
 51:     subvolumes t0-write-behind
 52: end-volume
 53:
 54: volume t0-io-cache
 55:     type performance/io-cache
 56:     subvolumes t0-read-ahead
 57: end-volume
 58:
 59: volume t0-quick-read
 60:     type performance/quick-read
 61:     subvolumes t0-io-cache
 62: end-volume
 63:
 64: volume t0-open-behind
 65:     type performance/open-behind
 66:     subvolumes t0-quick-read
 67: end-volume
 68:
 69: volume t0-md-cache
 70:     type performance/md-cache
 71:     subvolumes t0-open-behind
 72: end-volume
 73:
 74: volume t0
 75:     type debug/io-stats
 76:     option count-fop-hits off
 77:     option latency-measurement off
 78:     subvolumes t0-md-cache
 79: end-volume

+------------------------------------------------------------------------------+
[2014-08-12 19:20:50.823094] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 1-t0-client-0: changing port to 49152 (from 0)
[2014-08-12 19:20:50.826207] W [socket.c:514:__socket_rwv] 1-t0-client-0: readv failed (No data available)
[2014-08-12 19:20:50.828657] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 1-t0-client-1: changing port to 49152 (from 0)
[2014-08-12 19:20:50.828706] E [client-handshake.c:1742:client_query_portmap_cbk] 1-t0-client-2: failed to get the port number for remote subvolume. Please run 'gluster volume status' on server to see if brick process is running.
[2014-08-12 19:20:50.828798] W [socket.c:514:__socket_rwv] 1-t0-client-1: readv failed (No data available)
[2014-08-12 19:20:50.832987] W [socket.c:514:__socket_rwv] 1-t0-client-2: readv failed (No data available)
[2014-08-12 19:20:50.833005] I [client.c:2098:client_rpc_notify] 1-t0-client-2: disconnected
[2014-08-12 19:20:50.833017] E [client-handshake.c:1742:client_query_portmap_cbk] 1-t0-client-3: failed to get the port number for remote subvolume. Please run 'gluster volume status' on server to see if brick process is running.
[2014-08-12 19:20:50.833097] W [socket.c:514:__socket_rwv] 1-t0-client-3: readv failed (No data available)
[2014-08-12 19:20:50.833224] I [client.c:2098:client_rpc_notify] 1-t0-client-3: disconnected
[2014-08-12 19:20:50.833235] E [afr-common.c:3735:afr_notify] 1-t0-replicate-1: All subvolumes are down. Going offline until atleast one of them comes back up.
[2014-08-12 19:20:50.833318] I [client-handshake.c:1659:select_server_supported_programs] 1-t0-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:50.836164] I [client-handshake.c:1659:select_server_supported_programs] 1-t0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:50.836215] I [client-handshake.c:1456:client_setvolume_cbk] 1-t0-client-0: Connected to 192.168.30.107:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:20:50.836225] I [client-handshake.c:1468:client_setvolume_cbk] 1-t0-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:50.836309] I [afr-common.c:3698:afr_notify] 1-t0-replicate-0: Subvolume 't0-client-0' came back up; going online.
[2014-08-12 19:20:50.836482] I [client-handshake.c:450:client_set_lk_version_cbk] 1-t0-client-0: Server lk version = 1
[2014-08-12 19:20:50.836574] I [client-handshake.c:1456:client_setvolume_cbk] 1-t0-client-1: Connected to 192.168.30.108:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:20:50.836584] I [client-handshake.c:1468:client_setvolume_cbk] 1-t0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:50.842847] I [fuse-bridge.c:4771:fuse_graph_setup] 0-fuse: switched to graph 1
[2014-08-12 19:20:50.842874] I [client-handshake.c:450:client_set_lk_version_cbk] 1-t0-client-1: Server lk version = 1
[2014-08-12 19:20:50.882411] I [afr-common.c:2057:afr_set_root_inode_on_first_lookup] 1-t0-replicate-0: added root inode
[2014-08-12 19:20:50.887952] I [fuse-bridge.c:4001:fuse_migrate_fd_open] 0-glusterfs-fuse: migrated basefd (0x1bc4ffc) to newfd (0x1cb6b4c) (inode-gfid:eb10dffd-2314-4b9d-8048-1d7aa64ca006)(old-subvolume:t0-0 new-subvolume:t0-1)
[2014-08-12 19:20:50.887989] I [client.c:2164:notify] 0-t0-client-0: current graph is no longer active, destroying rpc_client
[2014-08-12 19:20:50.888029] I [client.c:2164:notify] 0-t0-client-1: current graph is no longer active, destroying rpc_client
[2014-08-12 19:20:50.888308] W [socket.c:514:__socket_rwv] 0-t0-client-0: readv failed (No data available)
[2014-08-12 19:20:50.888328] I [client.c:2098:client_rpc_notify] 0-t0-client-0: disconnected
[2014-08-12 19:20:50.888349] W [socket.c:514:__socket_rwv] 0-t0-client-1: readv failed (No data available)
[2014-08-12 19:20:50.888360] I [client.c:2098:client_rpc_notify] 0-t0-client-1: disconnected
[2014-08-12 19:20:50.888365] E [afr-common.c:3735:afr_notify] 0-t0-replicate-0: All subvolumes are down. Going offline until atleast one of them comes back up.
[2014-08-12 19:20:51.290477] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/protocol/client.so(client3_3_finodelk_cbk+0x157) [0x7f78820b74c7] (-->/usr/lib64/glusterfs/3.4.5/xlator/protocol/client.so(client_local_wipe+0x28) [0x7f78820abe18] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:20:51.300218] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:51.300239] W [dht-diskusage.c:45:dht_du_info_cbk] 1-t0-dht: failed to get disk info from t0-replicate-1
[2014-08-12 19:20:51.301085] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:52.302905] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:52.306924] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:52.306937] W [dht-diskusage.c:45:dht_du_info_cbk] 1-t0-dht: failed to get disk info from t0-replicate-1
[2014-08-12 19:20:52.307793] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:53.293216] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:53.297276] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:53.297289] W [dht-diskusage.c:45:dht_du_info_cbk] 1-t0-dht: failed to get disk info from t0-replicate-1
[2014-08-12 19:20:53.298122] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:54.279851] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:54.284084] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:54.284099] W [dht-diskusage.c:45:dht_du_info_cbk] 1-t0-dht: failed to get disk info from t0-replicate-1
[2014-08-12 19:20:54.285097] I [afr-common.c:3856:afr_local_init] 1-t0-replicate-1: no subvolumes up
[2014-08-12 19:20:54.536104] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 1-t0-client-2: changing port to 49153 (from 0)
[2014-08-12 19:20:54.536181] W [socket.c:514:__socket_rwv] 1-t0-client-2: readv failed (No data available)
[2014-08-12 19:20:54.542464] I [client-handshake.c:1659:select_server_supported_programs] 1-t0-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.542732] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 1-t0-client-3: changing port to 49153 (from 0)
[2014-08-12 19:20:54.542773] W [socket.c:514:__socket_rwv] 1-t0-client-3: readv failed (No data available)
[2014-08-12 19:20:54.545355] I [client-handshake.c:1456:client_setvolume_cbk] 1-t0-client-2: Connected to 192.168.30.107:49153, attached to remote volume '/bricks/brick2/brick'.
[2014-08-12 19:20:54.545368] I [client-handshake.c:1468:client_setvolume_cbk] 1-t0-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.545464] I [afr-common.c:3698:afr_notify] 1-t0-replicate-1: Subvolume 't0-client-2' came back up; going online.
[2014-08-12 19:20:54.548065] I [client-handshake.c:450:client_set_lk_version_cbk] 1-t0-client-2: Server lk version = 1
[2014-08-12 19:20:54.548085] I [client-handshake.c:1659:select_server_supported_programs] 1-t0-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.548354] I [client-handshake.c:1456:client_setvolume_cbk] 1-t0-client-3: Connected to 192.168.30.108:49153, attached to remote volume '/bricks/brick2/brick'.
[2014-08-12 19:20:54.548365] I [client-handshake.c:1468:client_setvolume_cbk] 1-t0-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.548511] I [client-handshake.c:450:client_set_lk_version_cbk] 1-t0-client-3: Server lk version = 1
[2014-08-12 19:20:55.283023] I [afr-common.c:2057:afr_set_root_inode_on_first_lookup] 1-t0-replicate-1: added root inode
[2014-08-12 19:20:55.286218] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:20:56.290222] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:20:56.290246] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:20:58.283783] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:20:58.283809] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:20:59.291060] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:20:59.291082] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:01.278644] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:01.278666] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:03.256118] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:03.256173] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:04.265057] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:04.265091] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:05.284792] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:05.284814] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:06.308370] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:06.308396] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:07.314231] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:07.314254] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:08.323802] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:08.323824] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:09.327628] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:09.327651] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:11.298759] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:11.298783] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:11.307919] W [client-rpc-fops.c:327:client3_3_mkdir_cbk] 1-t0-client-2: remote operation failed: Permission denied. Path: /102 (00000000-0000-0000-0000-000000000000)
[2014-08-12 19:21:11.307941] W [client-rpc-fops.c:327:client3_3_mkdir_cbk] 1-t0-client-3: remote operation failed: Permission denied. Path: /102 (00000000-0000-0000-0000-000000000000)
[2014-08-12 19:21:11.311245] I [afr-self-heal-entry.c:2253:afr_sh_entry_fix] 1-t0-replicate-1: /: Performing conservative merge
[2014-08-12 19:21:11.314866] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:11.314877] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:11.318024] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:11.318079] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:11.318450] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:11.318838] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:11.318866] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:11.318873] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:12.323238] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:12.323262] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:12.330677] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:12.330735] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:12.331183] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:12.331629] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:12.331640] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:12.331646] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:13.320084] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:13.325667] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:13.325689] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:13.326160] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:13.326624] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:13.326636] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:13.326642] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:13.328696] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:13.328708] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:14.308112] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:14.313591] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:14.313656] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:14.314105] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:14.314589] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:14.314601] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:14.314607] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:15.295866] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:15.295888] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:15.297570] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:15.302892] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:15.302970] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:15.303413] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:15.303922] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:15.303934] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:15.303940] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:16.285963] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.291397] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.291449] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.291916] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.292370] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.292382] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:16.292388] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:16.295473] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.295495] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.295939] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.296373] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.296385] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:16.296391] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:16.296403] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11460: /102/205/DATA => -1 (No such file or directory)
[2014-08-12 19:21:16.296487] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:16.299331] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.299352] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.299779] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.302997] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.303010] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:16.303015] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:16.303027] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11462: /102/205/DATA => -1 (No such file or directory)
[2014-08-12 19:21:16.303071] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:16.303713] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:16.303733] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:16.308624] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.308644] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.309124] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:16.309619] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:16.309631] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:16.309636] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.291402] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.296827] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.296846] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.297281] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.297727] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.297738] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.297744] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.300816] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.300836] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.301271] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.301683] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.301694] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.301700] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.301713] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11575: /102/207/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.301766] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.304602] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.304658] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.305028] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.305480] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.305502] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.305509] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.305520] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11577: /102/207/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.305565] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.307981] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:17.307993] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:17.311204] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.311221] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.311642] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.312082] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.312094] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.312099] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.315200] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.315221] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.315646] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.316008] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.316019] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.316024] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.316036] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11585: /102/208/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.316080] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.318925] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.318945] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.319375] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.319866] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.319878] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.319884] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.319895] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11587: /102/208/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.319946] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.324213] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.324233] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.324653] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.325112] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.325124] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.325129] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.328290] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.328308] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.328734] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.329087] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.329098] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.329103] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.329115] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11593: /102/209/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.329183] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.331974] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.332014] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.332377] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.332853] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.332865] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.332870] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:17.332882] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11595: /102/209/DATA => -1 (No such file or directory)
[2014-08-12 19:21:17.332926] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:17.337197] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.337241] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.337666] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:17.338042] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:17.338054] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:17.338059] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:18.321215] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:18.321238] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:18.323038] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.328440] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.328496] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.328941] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.329385] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.329396] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:18.329403] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:18.332399] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.332438] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.332892] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.333351] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.333362] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:18.333368] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:18.333381] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11708: /102/211/DATA => -1 (No such file or directory)
[2014-08-12 19:21:18.333436] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:18.336194] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.336247] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.336612] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.337098] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.337109] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:18.337115] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:18.337164] W [fuse-bridge.c:1869:fuse_create_cbk] 0-glusterfs-fuse: 11710: /102/211/DATA => -1 (No such file or directory)
[2014-08-12 19:21:18.337212] W [defaults.c:1291:default_release] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_create_done+0x149) [0x7f7881e40b39] (-->/usr/lib64/glusterfs/3.4.5/xlator/cluster/replicate.so(afr_local_cleanup+0x15c) [0x7f7881e7a82c] (-->/usr/lib64/libglusterfs.so.0(fd_unref+0x144) [0x7f7886cf81f4]))) 0-fuse: xlator does not implement release_cbk
[2014-08-12 19:21:18.341303] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.341321] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.341734] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:18.342111] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:18.342122] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:18.342128] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:19.345596] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:19.345623] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:19.353277] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:19.353301] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:19.353730] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:19.354117] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:19.354129] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:19.354170] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.
[2014-08-12 19:21:20.350479] I [dht-layout.c:726:dht_layout_dir_mismatch] 1-t0-dht: / - disk layout missing
[2014-08-12 19:21:20.350500] I [dht-common.c:623:dht_revalidate_cbk] 1-t0-dht: mismatching layouts for /
[2014-08-12 19:21:20.352207] W [client-rpc-fops.c:526:client3_3_stat_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:20.357433] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:20.357460] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:20.357915] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-2: remote operation failed: No such file or directory
[2014-08-12 19:21:20.358354] W [client-rpc-fops.c:1640:client3_3_entrylk_cbk] 1-t0-client-3: remote operation failed: No such file or directory
[2014-08-12 19:21:20.358366] I [afr-lk-common.c:1075:afr_lock_blocking] 1-t0-replicate-1: unable to lock on even one child
[2014-08-12 19:21:20.358371] I [afr-transaction.c:1113:afr_post_blocking_entrylk_cbk] 1-t0-replicate-1: Blocking entrylks failed.

root@gluster1:PRODUCTION:~> rpm -qa | grep gluster
glusterfs-3.4.5-1.el7.x86_64
glusterfs-libs-3.4.5-1.el7.x86_64
glusterfs-fuse-3.4.5-1.el7.x86_64
glusterfs-server-3.4.5-1.el7.x86_64
glusterfs-cli-3.4.5-1.el7.x86_64

root@gluster2:PRODUCTION:~> rpm -qa | grep gluster
glusterfs-3.4.5-1.el7.x86_64
glusterfs-libs-3.4.5-1.el7.x86_64
glusterfs-fuse-3.4.5-1.el7.x86_64
glusterfs-server-3.4.5-1.el7.x86_64
glusterfs-cli-3.4.5-1.el7.x86_64

smerrill@gluster-test:DEVELOPMENT:~> rpm -qa | grep gluster
glusterfs-fuse-3.4.5-1.el6.x86_64
glusterfs-3.4.5-1.el6.x86_64
glusterfs-libs-3.4.5-1.el6.x86_64

/var/log/glusterfs/glustershd.log from one of the servers:

[2014-08-12 18:51:35.034150] I [glusterfsd.c:1910:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.4.5 (/usr/sbin/glusterfs -s localhost --volfile-id gluster/glustershd -p /var/lib/glusterd/glustershd/run/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/8c96cd8469ab061c86fd0baa8fa84460.socket --xlator-option *replicate*.node-uuid=e88d4178-1045-4287-bfc2-5eec2a71c028)
[2014-08-12 18:51:35.039138] I [socket.c:3480:socket_init] 0-socket.glusterfsd: SSL support is NOT enabled
[2014-08-12 18:51:35.039202] I [socket.c:3495:socket_init] 0-socket.glusterfsd: using system polling thread
[2014-08-12 18:51:35.039381] I [socket.c:3480:socket_init] 0-glusterfs: SSL support is NOT enabled
[2014-08-12 18:51:35.039400] I [socket.c:3495:socket_init] 0-glusterfs: using system polling thread
[2014-08-12 18:51:35.053330] I [graph.c:239:gf_add_cmdline_options] 0-t0-replicate-0: adding option 'node-uuid' for volume 't0-replicate-0' with value 'e88d4178-1045-4287-bfc2-5eec2a71c028'
[2014-08-12 18:51:35.057551] I [socket.c:3480:socket_init] 0-t0-client-1: SSL support is NOT enabled
[2014-08-12 18:51:35.057582] I [socket.c:3495:socket_init] 0-t0-client-1: using system polling thread
[2014-08-12 18:51:35.058559] I [socket.c:3480:socket_init] 0-t0-client-0: SSL support is NOT enabled
[2014-08-12 18:51:35.058582] I [socket.c:3495:socket_init] 0-t0-client-0: using system polling thread
[2014-08-12 18:51:35.058617] I [client.c:2155:notify] 0-t0-client-0: parent translators are ready, attempting connect on transport
[2014-08-12 18:51:35.065448] I [client.c:2155:notify] 0-t0-client-1: parent translators are ready, attempting connect on transport
Given volfile:
+------------------------------------------------------------------------------+
  1: volume t0-client-0
  2:     type protocol/client
  3:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
  4:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
  5:     option transport-type tcp
  6:     option remote-subvolume /bricks/brick1/brick
  7:     option remote-host gluster1
  8: end-volume
  9:
 10: volume t0-client-1
 11:     type protocol/client
 12:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
 13:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
 14:     option transport-type tcp
 15:     option remote-subvolume /bricks/brick1/brick
 16:     option remote-host gluster2
 17: end-volume
 18:
 19: volume t0-replicate-0
 20:     type cluster/replicate
 21:     option iam-self-heal-daemon yes
 22:     option self-heal-daemon on
 23:     option entry-self-heal on
 24:     option data-self-heal on
 25:     option metadata-self-heal on
 26:     option background-self-heal-count 0
 27:     subvolumes t0-client-0 t0-client-1
 28: end-volume
 29:
 30: volume glustershd
 31:     type debug/io-stats
 32:     subvolumes t0-replicate-0
 33: end-volume

+------------------------------------------------------------------------------+
[2014-08-12 18:51:35.072476] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-1: changing port to 49152 (from 0)
[2014-08-12 18:51:35.072565] W [socket.c:514:__socket_rwv] 0-t0-client-1: readv failed (No data available)
[2014-08-12 18:51:35.078983] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-0: changing port to 49152 (from 0)
[2014-08-12 18:51:35.079045] W [socket.c:514:__socket_rwv] 0-t0-client-0: readv failed (No data available)
[2014-08-12 18:51:35.085547] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 18:51:35.085695] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 18:51:35.085926] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-1: Connected to 192.168.30.108:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 18:51:35.085950] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 18:51:35.086060] I [afr-common.c:3698:afr_notify] 0-t0-replicate-0: Subvolume 't0-client-1' came back up; going online.
[2014-08-12 18:51:35.086164] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-1: Server lk version = 1
[2014-08-12 18:51:35.086202] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-0: Connected to 192.168.30.107:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 18:51:35.086254] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 18:51:35.087355] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-0: Server lk version = 1
[2014-08-12 18:51:35.088320] I [afr-self-heald.c:1180:afr_dir_exclusive_crawl] 0-t0-replicate-0: Another crawl is in progress for t0-client-1
[2014-08-12 18:51:35.088515] I [afr-self-heald.c:1180:afr_dir_exclusive_crawl] 0-t0-replicate-0: Another crawl is in progress for t0-client-1
[2014-08-12 19:20:52.428455] I [glusterfsd-mgmt.c:58:mgmt_cbk_spec] 0-mgmt: Volume file changed
[2014-08-12 19:20:52.431595] I [glusterfsd-mgmt.c:1584:mgmt_getspec_cbk] 0-glusterfs: No change in volfile, continuing
[2014-08-12 19:20:53.018526] W [glusterfsd.c:1002:cleanup_and_exit] (-->/lib64/libc.so.6(clone+0x6d) [0x7fe2b110d3dd] (-->/lib64/libpthread.so.0(+0x7df3) [0x7fe2b17c4df3] (-->/usr/sbin/glusterfs(glusterfs_sigwaiter+0xe5) [0x7fe2b24b7f15]))) 0-: received signum (15), shutting down
[2014-08-12 19:20:54.032615] I [glusterfsd.c:1910:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.4.5 (/usr/sbin/glusterfs -s localhost --volfile-id gluster/glustershd -p /var/lib/glusterd/glustershd/run/glustershd.pid -l /var/log/glusterfs/glustershd.log -S /var/run/8c96cd8469ab061c86fd0baa8fa84460.socket --xlator-option *replicate*.node-uuid=e88d4178-1045-4287-bfc2-5eec2a71c028)
[2014-08-12 19:20:54.038057] I [socket.c:3480:socket_init] 0-socket.glusterfsd: SSL support is NOT enabled
[2014-08-12 19:20:54.038122] I [socket.c:3495:socket_init] 0-socket.glusterfsd: using system polling thread
[2014-08-12 19:20:54.038313] I [socket.c:3480:socket_init] 0-glusterfs: SSL support is NOT enabled
[2014-08-12 19:20:54.038331] I [socket.c:3495:socket_init] 0-glusterfs: using system polling thread
[2014-08-12 19:20:54.052871] I [graph.c:239:gf_add_cmdline_options] 0-t0-replicate-1: adding option 'node-uuid' for volume 't0-replicate-1' with value 'e88d4178-1045-4287-bfc2-5eec2a71c028'
[2014-08-12 19:20:54.052900] I [graph.c:239:gf_add_cmdline_options] 0-t0-replicate-0: adding option 'node-uuid' for volume 't0-replicate-0' with value 'e88d4178-1045-4287-bfc2-5eec2a71c028'
[2014-08-12 19:20:54.061469] I [socket.c:3480:socket_init] 0-t0-client-3: SSL support is NOT enabled
[2014-08-12 19:20:54.061525] I [socket.c:3495:socket_init] 0-t0-client-3: using system polling thread
[2014-08-12 19:20:54.062621] I [socket.c:3480:socket_init] 0-t0-client-2: SSL support is NOT enabled
[2014-08-12 19:20:54.062644] I [socket.c:3495:socket_init] 0-t0-client-2: using system polling thread
[2014-08-12 19:20:54.063744] I [socket.c:3480:socket_init] 0-t0-client-1: SSL support is NOT enabled
[2014-08-12 19:20:54.063766] I [socket.c:3495:socket_init] 0-t0-client-1: using system polling thread
[2014-08-12 19:20:54.064859] I [socket.c:3480:socket_init] 0-t0-client-0: SSL support is NOT enabled
[2014-08-12 19:20:54.064881] I [socket.c:3495:socket_init] 0-t0-client-0: using system polling thread
[2014-08-12 19:20:54.064929] I [client.c:2155:notify] 0-t0-client-0: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:54.071893] I [client.c:2155:notify] 0-t0-client-1: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:54.078871] I [client.c:2155:notify] 0-t0-client-2: parent translators are ready, attempting connect on transport
[2014-08-12 19:20:54.085209] I [client.c:2155:notify] 0-t0-client-3: parent translators are ready, attempting connect on transport
Given volfile:
+------------------------------------------------------------------------------+
  1: volume t0-client-0
  2:     type protocol/client
  3:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
  4:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
  5:     option transport-type tcp
  6:     option remote-subvolume /bricks/brick1/brick
  7:     option remote-host gluster1
  8: end-volume
  9:
 10: volume t0-client-1
 11:     type protocol/client
 12:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
 13:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
 14:     option transport-type tcp
 15:     option remote-subvolume /bricks/brick1/brick
 16:     option remote-host gluster2
 17: end-volume
 18:
 19: volume t0-client-2
 20:     type protocol/client
 21:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
 22:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
 23:     option transport-type tcp
 24:     option remote-subvolume /bricks/brick2/brick
 25:     option remote-host gluster1
 26: end-volume
 27:
 28: volume t0-client-3
 29:     type protocol/client
 30:     option password a3fd2c37-13e7-4d65-b9d1-123f2aad05e8
 31:     option username 40a9a10e-1eb7-4f2b-92ea-06dd7086d767
 32:     option transport-type tcp
 33:     option remote-subvolume /bricks/brick2/brick
 34:     option remote-host gluster2
 35: end-volume
 36:
 37: volume t0-replicate-0
 38:     type cluster/replicate
 39:     option iam-self-heal-daemon yes
 40:     option self-heal-daemon on
 41:     option entry-self-heal on
 42:     option data-self-heal on
 43:     option metadata-self-heal on
 44:     option background-self-heal-count 0
 45:     subvolumes t0-client-0 t0-client-1
 46: end-volume
 47:
 48: volume t0-replicate-1
 49:     type cluster/replicate
 50:     option iam-self-heal-daemon yes
 51:     option self-heal-daemon on
 52:     option entry-self-heal on
 53:     option data-self-heal on
 54:     option metadata-self-heal on
 55:     option background-self-heal-count 0
 56:     subvolumes t0-client-2 t0-client-3
 57: end-volume
 58:
 59: volume glustershd
 60:     type debug/io-stats
 61:     subvolumes t0-replicate-0 t0-replicate-1
 62: end-volume

+------------------------------------------------------------------------------+
[2014-08-12 19:20:54.092192] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-1: changing port to 49152 (from 0)
[2014-08-12 19:20:54.092290] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-0: changing port to 49152 (from 0)
[2014-08-12 19:20:54.092332] W [socket.c:514:__socket_rwv] 0-t0-client-1: readv failed (No data available)
[2014-08-12 19:20:54.098684] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-3: changing port to 49153 (from 0)
[2014-08-12 19:20:54.098764] W [socket.c:514:__socket_rwv] 0-t0-client-0: readv failed (No data available)
[2014-08-12 19:20:54.105113] W [socket.c:514:__socket_rwv] 0-t0-client-3: readv failed (No data available)
[2014-08-12 19:20:54.111534] I [rpc-clnt.c:1676:rpc_clnt_reconfig] 0-t0-client-2: changing port to 49153 (from 0)
[2014-08-12 19:20:54.111665] W [socket.c:514:__socket_rwv] 0-t0-client-2: readv failed (No data available)
[2014-08-12 19:20:54.118129] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.118286] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-3: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.118405] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.118502] I [client-handshake.c:1659:select_server_supported_programs] 0-t0-client-2: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2014-08-12 19:20:54.118609] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-1: Connected to 192.168.30.108:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:20:54.118630] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.118734] I [afr-common.c:3698:afr_notify] 0-t0-replicate-0: Subvolume 't0-client-1' came back up; going online.
[2014-08-12 19:20:54.118821] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-1: Server lk version = 1
[2014-08-12 19:20:54.118900] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-0: Connected to 192.168.30.107:49152, attached to remote volume '/bricks/brick1/brick'.
[2014-08-12 19:20:54.118917] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.119030] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-2: Connected to 192.168.30.107:49153, attached to remote volume '/bricks/brick2/brick'.
[2014-08-12 19:20:54.119049] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-2: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.119114] I [afr-common.c:3698:afr_notify] 0-t0-replicate-1: Subvolume 't0-client-2' came back up; going online.
[2014-08-12 19:20:54.119184] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-0: Server lk version = 1
[2014-08-12 19:20:54.119280] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-2: Server lk version = 1
[2014-08-12 19:20:54.120201] I [afr-self-heald.c:1180:afr_dir_exclusive_crawl] 0-t0-replicate-0: Another crawl is in progress for t0-client-1
[2014-08-12 19:20:54.120398] I [afr-self-heald.c:1180:afr_dir_exclusive_crawl] 0-t0-replicate-0: Another crawl is in progress for t0-client-1
[2014-08-12 19:20:54.123084] I [client-handshake.c:1456:client_setvolume_cbk] 0-t0-client-3: Connected to 192.168.30.108:49153, attached to remote volume '/bricks/brick2/brick'.
[2014-08-12 19:20:54.123126] I [client-handshake.c:1468:client_setvolume_cbk] 0-t0-client-3: Server and Client lk-version numbers are not same, reopening the fds
[2014-08-12 19:20:54.123386] I [client-handshake.c:450:client_set_lk_version_cbk] 0-t0-client-3: Server lk version = 1