2018-04-04_22-17-03.txt

[2018-04-05 05:13:37.509199] I [cli.c:767:main] 0-cli: Started running /usr/sbin/gluster with version 4.0.1
[2018-04-05 05:13:37.566727] I [MSGID: 101190] [event-epoll.c:609:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2018-04-05 05:13:37.566865] I [socket.c:2625:socket_event_handler] 0-transport: EPOLLERR - disconnecting now

==> glfsheal-data.log <==
[2018-04-05 05:13:37.578495] I [MSGID: 104045] [glfs-master.c:91:notify] 0-gfapi: New graph 70796c6f-6e2d-3238-3431-352d32303138 (0) coming up
[2018-04-05 05:13:37.578610] I [MSGID: 114020] [client.c:2300:notify] 0-data-client-0: parent translators are ready, attempting connect on transport
[2018-04-05 05:13:37.579082] I [MSGID: 114020] [client.c:2300:notify] 0-data-client-1: parent translators are ready, attempting connect on transport
[2018-04-05 05:13:37.579169] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-0: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.579335] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-0: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.579471] I [MSGID: 114020] [client.c:2300:notify] 0-data-client-2: parent translators are ready, attempting connect on transport
[2018-04-05 05:13:37.579449] I [rpc-clnt.c:2071:rpc_clnt_reconfig] 0-data-client-0: changing port to 49152 (from 0)
[2018-04-05 05:13:37.579623] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-1: error returned while attempting to connect to host:(null), port:0
Final graph:
+------------------------------------------------------------------------------+
  1: volume data-client-0
  2:     type protocol/client
  3:     option ping-timeout 5
  4:     option remote-host pylon
  5:     option remote-subvolume /mnt/pylon_block1/data
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username XXXXXXXXXXXXXXXXX
  9:     option password XXXXXXXXXXXXXXXXX
 10:     option filter-O_DIRECT enable
 11:     option event-threads 4
 12:     option transport.tcp-user-timeout 0
 13:     option transport.socket.keepalive-time 20
 14:     option transport.socket.keepalive-interval 2
 15:     option transport.socket.keepalive-count 9
 16:     option send-gids true
 17: end-volume
 18:
 19: volume data-client-1
 20:     type protocol/client
 21:     option ping-timeout 5
 22:     option remote-host pylon
 23:     option remote-subvolume /mnt/pylon_block2/data
 24:     option transport-type socket
 25:     option transport.address-family inet
 26:     option username XXXXXXXXXXXXXXXXX
 27:     option password XXXXXXXXXXXXXXXXX
[2018-04-05 05:13:37.579862] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-2: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.580573] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-2: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.579839] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-1: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.580745] I [rpc-clnt.c:2071:rpc_clnt_reconfig] 0-data-client-2: changing port to 49154 (from 0)
[2018-04-05 05:13:37.580791] I [rpc-clnt.c:2071:rpc_clnt_reconfig] 0-data-client-1: changing port to 49153 (from 0)
[2018-04-05 05:13:37.581042] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-2: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.581099] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-1: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.581185] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-2: error returned while attempting to connect to host:(null), port:0
[2018-04-05 05:13:37.581506] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-1: error returned while attempting to connect to host:(null), port:0

==> bricks/mnt-pylon_block3-data.log <==
[2018-04-05 05:13:37.581448] I [addr.c:55:compare_addr_and_update] 0-/mnt/pylon_block3/data: allowed = "*", received addr = "192.168.128.68"
[2018-04-05 05:13:37.581600] I [login.c:76:gf_auth] 0-auth/login: allowed user names: XXXXXXXXXXXXXXXXX
[2018-04-05 05:13:37.581614] I [MSGID: 115029] [server-handshake.c:762:server_setvolume] 0-data-server: accepted client from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-2-RECON_NO:-0 (version: 4.0.1)

==> bricks/mnt-pylon_block2-data.log <==
[2018-04-05 05:13:37.581640] I [addr.c:55:compare_addr_and_update] 0-/mnt/pylon_block2/data: allowed = "*", received addr = "192.168.128.68"
[2018-04-05 05:13:37.581664] I [login.c:76:gf_auth] 0-auth/login: allowed user names: XXXXXXXXXXXXXXXXX
[2018-04-05 05:13:37.581675] I [MSGID: 115029] [server-handshake.c:762:server_setvolume] 0-data-server: accepted client from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-1-RECON_NO:-0 (version: 4.0.1)

==> glfsheal-data.log <==
[2018-04-05 05:13:37.582069] I [MSGID: 114046] [client-handshake.c:1176:client_setvolume_cbk] 0-data-client-1: Connected to data-client-1, attached to remote volume '/mnt/pylon_block2/data'.
[2018-04-05 05:13:37.582096] I [MSGID: 108005] [afr-common.c:5079:__afr_handle_child_up_event] 0-data-replicate-0: Subvolume 'data-client-1' came back up; going online.
[2018-04-05 05:13:37.582479] I [MSGID: 114046] [client-handshake.c:1176:client_setvolume_cbk] 0-data-client-2: Connected to data-client-2, attached to remote volume '/mnt/pylon_block3/data'.
[2018-04-05 05:13:37.580105] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-0: error returned while attempting to connect to host:(null), port:0
 28:     option filter-O_DIRECT enable
 29:     option event-threads 4
 30:     option transport.tcp-user-timeout 0
 31:     option transport.socket.keepalive-time 20
 32:     option transport.socket.keepalive-interval 2
 33:     option transport.socket.keepalive-count 9
 34:     option send-gids true
 35: end-volume
 36:
 37: volume data-client-2
 38:     type protocol/client
 39:     option opversion 40000
 40:     option clnt-lk-version 1
 41:     option volfile-checksum 0
 42:     option volfile-key data
 43:     option client-version 4.0.1
 44:     option process-name gfapi.glfsheal
 45:     option process-uuid CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-2-RECON_NO:-0
 46:     option fops-version 1298437
 47:     option ping-timeout 5
 48:     option remote-host pylon
 49:     option remote-subvolume /mnt/pylon_block3/data
 50:     option transport-type socket
 51:     option transport.address-family inet
 52:     option username XXXXXXXXXXXXXXXXX
 53:     option password XXXXXXXXXXXXXXXXX
 54:     option filter-O_DIRECT enable
 55:     option event-threads 4
 56:     option transport.tcp-user-timeout 0
 57:     option transport.socket.keepalive-time 20
 58:     option transport.socket.keepalive-interval 2
 59:     option transport.socket.keepalive-count 9
 60:     option send-gids true
 61: end-volume
 62:
 63: volume data-replicate-0
 64:     type cluster/replicate
 65:     option background-self-heal-count 0
 66:     option afr-pending-xattr data-client-0,data-client-1,data-client-2
 67:     option read-hash-mode 0
 68:     option quorum-type fixed
 69:     option quorum-count 1
 70:     option shd-max-threads 12
 71:     option use-compound-fops off
 72:     subvolumes data-client-0 data-client-1 data-client-2
 73: end-volume
 74:
 75: volume data-readdir-ahead-0
 76:     type performance/readdir-ahead
 77:     option parallel-readdir on
 78:     option rda-request-size 131072
 79:     option rda-cache-limit 256MB
 80:     subvolumes data-replicate-0
 81: end-volume
 82:
 83: volume data-dht
 84:     type cluster/distribute
 85:     option lookup-unhashed auto
 86:     option lookup-optimize on
 87:     option readdir-optimize on
 88:     option lock-migration off
 89:     subvolumes data-readdir-ahead-0
 90: end-volume
 91:
 92: volume data-write-behind
 93:     type performance/write-behind
 94:     option strict-O_DIRECT on
 95:     subvolumes data-dht
 96: end-volume
 97:
 98: volume data-io-cache
 99:     type performance/io-cache
100:     option cache-size 1GB
101:     subvolumes data-write-behind
102: end-volume
103:
104: volume data-quick-read
105:     type performance/quick-read
106:     option cache-size 1GB
107:     subvolumes data-io-cache
108: end-volume
109:
110: volume data-open-behind
111:     type performance/open-behind
112:     subvolumes data-quick-read
113: end-volume
115: volume data-nl-cache
116:     type performance/nl-cache
117:     option nl-cache-limit 10MB
118:     option nl-cache-timeout 600
119:     subvolumes data-open-behind
120: end-volume
121:
122: volume data-md-cache
123:     type performance/md-cache
124:     option md-cache-timeout 600
125:     option cache-invalidation on
[2018-04-05 05:13:37.583249] I [MSGID: 108002] [afr-common.c:5376:afr_notify] 0-data-replicate-0: Client-quorum is met
[2018-04-05 05:13:37.583061] W [rpc-clnt.c:1739:rpc_clnt_submit] 0-data-client-0: error returned while attempting to connect to host:(null), port:0

==> bricks/mnt-pylon_block1-data.log <==
[2018-04-05 05:13:37.585203] I [addr.c:55:compare_addr_and_update] 0-/mnt/pylon_block1/data: allowed = "*", received addr = "192.168.128.68"
[2018-04-05 05:13:37.585228] I [login.c:76:gf_auth] 0-auth/login: allowed user names: XXXXXXXXXXXXXXXXX
[2018-04-05 05:13:37.585241] I [MSGID: 115029] [server-handshake.c:762:server_setvolume] 0-data-server: accepted client from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-0-RECON_NO:-0 (version: 4.0.1)

==> glfsheal-data.log <==
126:     subvolumes data-nl-cache
127: end-volume
128:
129: volume data
130:     type debug/io-stats
131:     option log-level INFO
132:     option latency-measurement off
133:     option count-fop-hits off
134:     subvolumes data-md-cache
135: end-volume
[2018-04-05 05:13:37.585720] I [MSGID: 114046] [client-handshake.c:1176:client_setvolume_cbk] 0-data-client-0: Connected to data-client-0, attached to remote volume '/mnt/pylon_block1/data'.
136:
137: volume meta-autoload
138:     type meta
139:     subvolumes data
140: end-volume
141:
+------------------------------------------------------------------------------+
[2018-04-05 05:13:37.590271] I [MSGID: 108031] [afr-common.c:2530:afr_local_discovery_cbk] 0-data-replicate-0: selecting local read_child data-client-2
[2018-04-05 05:13:37.590506] I [MSGID: 108031] [afr-common.c:2530:afr_local_discovery_cbk] 0-data-replicate-0: selecting local read_child data-client-0
[2018-04-05 05:13:37.590705] I [MSGID: 108031] [afr-common.c:2530:afr_local_discovery_cbk] 0-data-replicate-0: selecting local read_child data-client-1
[2018-04-05 05:13:37.591038] I [MSGID: 109005] [dht-selfheal.c:2328:dht_selfheal_directory] 0-data-dht: Directory selfheal failed: Unable to form layout for directory /
[2018-04-05 05:13:37.591142] I [MSGID: 104041] [glfs-resolve.c:972:__glfs_active_subvol] 0-data: switched to graph 70796c6f-6e2d-3238-3431-352d32303138 (0)

==> bricks/mnt-pylon_block1-data.log <==
[2018-04-05 05:13:37.596784] I [MSGID: 115036] [server.c:475:server_rpc_notify] 0-data-server: disconnecting connection from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-0-RECON_NO:-0
[2018-04-05 05:13:37.596918] I [MSGID: 101055] [client_t.c:444:gf_client_unref] 0-data-server: Shutting down connection CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-0-RECON_NO:-0

==> bricks/mnt-pylon_block2-data.log <==
[2018-04-05 05:13:37.597115] I [MSGID: 115036] [server.c:475:server_rpc_notify] 0-data-server: disconnecting connection from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-1-RECON_NO:-0
[2018-04-05 05:13:37.597202] I [MSGID: 101055] [client_t.c:444:gf_client_unref] 0-data-server: Shutting down connection CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-1-RECON_NO:-0

==> bricks/mnt-pylon_block3-data.log <==
[2018-04-05 05:13:37.597386] I [MSGID: 115036] [server.c:475:server_rpc_notify] 0-data-server: disconnecting connection from CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-2-RECON_NO:-0
[2018-04-05 05:13:37.597498] I [MSGID: 101055] [client_t.c:444:gf_client_unref] 0-data-server: Shutting down connection CTX_ID:9a06db98-67a9-4bee-a8df-af9e1da459ad-GRAPH_ID:0-PID:28415-HOST:pylon-PC_NAME:data-client-2-RECON_NO:-0

==> cli.log <==
[2018-04-05 05:13:37.597491] I [input.c:31:cli_batch] 0-: Exiting with: 0