Skip to content

Instantly share code, notes, and snippets.

@charz

charz/check_object_0629.md

Last active August 29, 2015 14:23
Show Gist options
  • Save charz/4326fbf035b1904d531c to your computer and use it in GitHub Desktop.
Save charz/4326fbf035b1904d531c to your computer and use it in GitHub Desktop.
Download ZIP
FA is missing .durable file and can't retrieve object from object server in 40% disk failure in EC6+4
Raw
check_object_0629.md

Available nodes

## qa01
swiftqa@swiftqa01:~$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
total 28172
drwxr-xr-x 2 swift swift       44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#9.data

## qa04
swiftqa@swiftqa04:~$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
/srv/node/d53/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#7.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

/srv/node/d54/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#2.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

## qa05
[swiftqa@swiftqa05 ~]$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
/srv/node/d30/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x. 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x. 3 swift swift       53 Jun 25 12:55 ..
-rw-------. 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#0.data
-rw-r--r--. 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

/srv/node/d31/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x. 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x. 3 swift swift       53 Jun 25 12:55 ..
-rw-------. 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#5.data
-rw-r--r--. 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

Unavailable nodes

## qa02 (unmount disks)
swiftqa@swiftqa02:/var/log/swift$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
/srv/node/d5/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#4.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

/srv/node/d6/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#8.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

## qa03 (offline)
swiftqa@swiftqa03:~$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
/srv/node/d17/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       45 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#6.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

/srv/node/d19/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#3.data
-rw-r--r-- 1 swift swift        0 Jun 25 12:55 1435236883.13246.durable

Get object

swift --debug download ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation objects_000066 --no-download
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): 192.168.11.11
DEBUG:requests.packages.urllib3.connectionpool:"GET /auth/v1.0 HTTP/1.1" 200 0
DEBUG:swiftclient:REQ: curl -i http://192.168.11.11/auth/v1.0 -X GET
DEBUG:swiftclient:RESP STATUS: 200 OK
DEBUG:swiftclient:RESP HEADERS: [('content-length', '0'), ('x-trans-id', 'tx13c1fe1645314f10b37e2-005590c761'), ('x-auth-token', 'AUTH_tk159adee260584353bbc853c8d782aa08'), ('connection', 'keep-alive'), ('x-storage-token', 'AUTH_tk159adee260584353bbc853c8d782aa08'), ('date', 'Mon, 29 Jun 2015 04:19:46 GMT'), ('x-storage-url', 'http://192.168.11.11/v1/AUTH_test'), ('content-type', 'text/html; charset=UTF-8')]
INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): 192.168.11.11
DEBUG:requests.packages.urllib3.connectionpool:"GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 HTTP/1.1" 404 70
INFO:swiftclient:REQ: curl -i http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 -X GET -H "X-Auth-Token: AUTH_tk159adee260584353bbc853c8d782aa08"
INFO:swiftclient:RESP STATUS: 404 Not Found
INFO:swiftclient:RESP HEADERS: [('date', 'Mon, 29 Jun 2015 04:19:49 GMT'), ('content-length', '70'), ('content-type', 'text/html; charset=UTF-8'), ('connection', 'keep-alive'), ('x-trans-id', 'tx809d4a3b453f456fbe08c-005590c762')]
INFO:swiftclient:RESP BODY: <html><h1>Not Found</h1><p>The resource could not be found.</p></html>
ERROR:swiftclient:Object GET failed: http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 404 Not Found  [first 60 chars of response] <html><h1>Not Found</h1><p>The resource could not be found.<
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/swiftclient/client.py", line 1261, in _retry
    rv = func(self.url, self.token, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/swiftclient/client.py", line 875, in get_object
    http_response_content=body)
ClientException: Object GET failed: http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 404 Not Found  [first 60 chars of response] <html><h1>Not Found</h1><p>The resource could not be found.<
Object 'ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066' not found

QA01

swiftqa@swiftqa01:/var/log/swift$ grep "tx809d4a3b453f456fbe08c-005590c762" /var/log/swift/all.log*
/var/log/swift/all.log.2:Jun 29 04:19:46 localhost.localdomain proxy-server: STDERR: ERROR:root:Timeout connecting to memcached: 192.168.12.13:11211 (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:47 localhost.localdomain proxy-server: ERROR with Account server 192.168.12.13:6002/d14 re: Trying to HEAD /v1/AUTH_test: ConnectionTimeout (0.5s) (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:47 localhost.localdomain account-server: 192.168.12.11 - - [29/Jun/2015:04:19:47 +0000] "HEAD /d41/2303/AUTH_test" 204 - "HEAD http://192.168.11.11/v1/AUTH_test" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0293 "-" 24564 -
/var/log/swift/all.log.2:Jun 29 04:19:47 localhost.localdomain proxy-server: STDERR: ERROR:root:Timeout connecting to memcached: 192.168.12.13:11211 (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:47 localhost.localdomain proxy-server: - - 29/Jun/2015/04/19/47 HEAD /v1/AUTH_test HTTP/1.0 204 - Swift - - - - tx809d4a3b453f456fbe08c-005590c762 - 0.8490 RL - 1435551586.756392956 1435551587.605436087 -
/var/log/swift/all.log.2:Jun 29 04:19:47 localhost.localdomain proxy-server: STDERR: ERROR:root:Timeout connecting to memcached: 192.168.12.13:11211 (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain proxy-server: ERROR with Container server 192.168.12.13:6001/d15 re: Trying to HEAD /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation: ConnectionTimeout (0.5s) (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain proxy-server: ERROR Insufficient Storage 192.168.12.12:6001/d7 (txn: tx809d4a3b453f456fbe08c-005590c762)
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain container-server: 192.168.12.11 - - [29/Jun/2015:04:19:48 +0000] "HEAD /d47/3153/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation" 204 - "HEAD http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0176 "-" 24462 3
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d41/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0011 "-" 24510 3
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0010 "-" 24510 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d40/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0010 "-" 24510 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d42/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0010 "-" 24510 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost.localdomain proxy-server: 192.168.11.41 192.168.11.41 29/Jun/2015/04/19/49 GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 HTTP/1.0 404 - python-swiftclient-2.4.0 test%2CAUTH_tk159a... - 70 - tx809d4a3b453f456fbe08c-005590c762 - 3.0923 - - 1435551586.446428061 1435551589.538718939 3

##QA02 (all disk umount)

swiftqa@swiftqa02:/var/log/swift$ grep "tx809d4a3b453f456fbe08c-005590c762" /var/log/swift/all.log*
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain container-server: 192.168.12.11 - - [29/Jun/2015:04:19:48 +0000] "HEAD /d7/3153/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation" 507 - "HEAD http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0007 "-" 6765 -
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d6/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 507 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0007 "-" 6816 3
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000]

##QA03 (Offline)

##QA04

swiftqa@swiftqa04:/var/log/swift$ grep "tx809d4a3b453f456fbe08c-005590c762" /var/log/swift/all.log*
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d54/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 200 28841598 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0017 "-" 8210 3
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d53/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 200 28841598 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0242 "-" 8213 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d52/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0009 "-" 8216 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost.localdomain object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d55/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0009 "-" 8210 3

##QA05

[swiftqa@swiftqa05 swift]$ grep "tx809d4a3b453f456fbe08c-005590c762" /var/log/swift/all.log*
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d31/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 200 28841598 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0026 "-" 8945 3
/var/log/swift/all.log.2:Jun 29 04:19:48 localhost object-server: 192.168.11.11 - - [29/Jun/2015:04:19:48 +0000] "GET /d30/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 200 28841598 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0026 "-" 8954 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d29/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0014 "-" 8958 3
/var/log/swift/all.log.2:Jun 29 04:19:49 localhost object-server: 192.168.11.11 - - [29/Jun/2015:04:19:49 +0000] "GET /d28/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx809d4a3b453f456fbe08c-005590c762" "proxy-server 24549" 0.0014 "-" 8954 3
Raw
cli.md

get object from cli

swiftstack@bm01:/data/charz/failure_test$ swift stat ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation objects_000066 
       Account: AUTH_test
     Container: ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation
        Object: objects_000066
  Content Type: application/octet-stream
Content Length: 172970057
 Last Modified: Thu, 25 Jun 2015 12:54:44 GMT
          ETag: f32c6509a66ffa7ccfa5484ed5e3d3fe
 Accept-Ranges: bytes
    Connection: keep-alive
   X-Timestamp: 1435236883.13246
    X-Trans-Id: tx1075908baf3543aeaf59c-00558d1721


swiftstack@bm01:/data/charz/failure_test$ swift download ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation objects_000066 -v
Object 'ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066' not found

logs

Jun 26 09:11:28 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:28 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx3cb8fe139c154de7ae3ed-00558d173e" "proxy-server 24546" 0.0011 "-" 24512 3
Jun 26 09:11:28 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:28 +0000] "GET /d41/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx3cb8fe139c154de7ae3ed-00558d173e" "proxy-server 24546" 0.0012 "-" 24515 3
Jun 26 09:11:29 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d17 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:29 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d19 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:29 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:29 +0000] "GET /d40/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx3cb8fe139c154de7ae3ed-00558d173e" "proxy-server 24546" 0.0009 "-" 24512 3
Jun 26 09:11:29 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d18 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:29 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:29 +0000] "GET /d42/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx3cb8fe139c154de7ae3ed-00558d173e" "proxy-server 24546" 0.0010 "-" 24512 3
Jun 26 09:11:29 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d16 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:29 localhost.localdomain proxy-server: 192.168.11.41 192.168.11.41 26/Jun/2015/09/11/29 GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 HTTP/1.0 503 - python-swiftclient-2.4.0 test%2CAUTH_tke893... - 118 - tx3cb8fe139c154de7ae3ed-00558d173e - 2.9685 - - 1435309886.810317993 1435309889.778805017 3
Jun 26 09:11:31 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:31 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx359db4a5d7e947c1ba5aa-00558d1742" "proxy-server 24546" 0.0011 "-" 24514 3
Jun 26 09:11:31 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:31 +0000] "GET /d41/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx359db4a5d7e947c1ba5aa-00558d1742" "proxy-server 24546" 0.0013 "-" 24514 3
Jun 26 09:11:31 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d17 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:31 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d19 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: ConnectionTimeout (0.5s)
Jun 26 09:11:31 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:31 +0000] "GET /d40/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx359db4a5d7e947c1ba5aa-00558d1742" "proxy-server 24546" 0.0010 "-" 24508 3
Jun 26 09:11:31 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d18 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: Host unreachable
Jun 26 09:11:31 localhost.localdomain proxy-server: ERROR with Object server 192.168.11.13:6000/d16 re: Trying to GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066: Host unreachable
Jun 26 09:11:31 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:09:11:31 +0000] "GET /d42/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx359db4a5d7e947c1ba5aa-00558d1742" "proxy-server 24546" 0.0009 "-" 24512 3
Jun 26 09:11:31 localhost.localdomain proxy-server: 192.168.11.41 192.168.11.41 26/Jun/2015/09/11/31 GET /v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066 HTTP/1.0 404 - python-swiftclient-2.4.0 test%2CAUTH_tke893... - 70 - tx359db4a5d7e947c1ba5aa-00558d1742 - 0.9764 - - 1435309890.784748077 1435309891.761162043 3
Raw
get_node.md

Two servers are offline (40% disks failure)

  • 192.168.11.12
  • 192.168.11.13

Get node

swiftqa@swiftqa01:~$ swift-get-nodes -P ec64-isolation AUTH_test ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation objects_000066

Account  	AUTH_test
Container	ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation
Object   	objects_000066


Partition	2170
Hash     	87a0767a119155053615399f030dd972

Server:Port Device	192.168.11.15:6000 d30
Server:Port Device	192.168.11.11:6000 d43
Server:Port Device	192.168.11.14:6000 d54
Server:Port Device	192.168.11.13:6000 d19
Server:Port Device	192.168.11.12:6000 d5
Server:Port Device	192.168.11.15:6000 d31
Server:Port Device	192.168.11.13:6000 d17
Server:Port Device	192.168.11.14:6000 d53
Server:Port Device	192.168.11.12:6000 d6
Server:Port Device	192.168.11.11:6000 d41
Server:Port Device	192.168.11.15:6000 d29	 [Handoff]
Server:Port Device	192.168.11.13:6000 d18	 [Handoff]
Server:Port Device	192.168.11.12:6000 d4	 [Handoff]
Server:Port Device	192.168.11.11:6000 d40	 [Handoff]
Server:Port Device	192.168.11.14:6000 d52	 [Handoff]
Server:Port Device	192.168.11.13:6000 d16	 [Handoff]
Server:Port Device	192.168.11.15:6000 d28	 [Handoff]
Server:Port Device	192.168.11.11:6000 d42	 [Handoff]
Server:Port Device	192.168.11.14:6000 d55	 [Handoff]
Server:Port Device	192.168.11.12:6000 d7	 [Handoff]


curl -I -XHEAD "http://192.168.11.15:6000/d30/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.11:6000/d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.14:6000/d54/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.13:6000/d19/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.12:6000/d5/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.15:6000/d31/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.13:6000/d17/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.14:6000/d53/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.12:6000/d6/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.11:6000/d41/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
curl -I -XHEAD "http://192.168.11.15:6000/d29/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.13:6000/d18/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.12:6000/d4/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.11:6000/d40/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.14:6000/d52/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.13:6000/d16/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.15:6000/d28/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.11:6000/d42/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.14:6000/d55/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]
curl -I -XHEAD "http://192.168.11.12:6000/d7/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3" # [Handoff]


Use your own device location of servers:
such as "export DEVICE=/srv/node"
ssh 192.168.11.15 "ls -lah ${DEVICE:-/srv/node*}/d30/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.11 "ls -lah ${DEVICE:-/srv/node*}/d43/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.14 "ls -lah ${DEVICE:-/srv/node*}/d54/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.13 "ls -lah ${DEVICE:-/srv/node*}/d19/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.12 "ls -lah ${DEVICE:-/srv/node*}/d5/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.15 "ls -lah ${DEVICE:-/srv/node*}/d31/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.13 "ls -lah ${DEVICE:-/srv/node*}/d17/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.14 "ls -lah ${DEVICE:-/srv/node*}/d53/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.12 "ls -lah ${DEVICE:-/srv/node*}/d6/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.11 "ls -lah ${DEVICE:-/srv/node*}/d41/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh 192.168.11.15 "ls -lah ${DEVICE:-/srv/node*}/d29/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.13 "ls -lah ${DEVICE:-/srv/node*}/d18/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.12 "ls -lah ${DEVICE:-/srv/node*}/d4/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.11 "ls -lah ${DEVICE:-/srv/node*}/d40/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.14 "ls -lah ${DEVICE:-/srv/node*}/d52/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.13 "ls -lah ${DEVICE:-/srv/node*}/d16/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.15 "ls -lah ${DEVICE:-/srv/node*}/d28/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.11 "ls -lah ${DEVICE:-/srv/node*}/d42/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.14 "ls -lah ${DEVICE:-/srv/node*}/d55/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh 192.168.11.12 "ls -lah ${DEVICE:-/srv/node*}/d7/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]

note: `/srv/node*` is used as default value of `devices`, the real value is set in the config file on each storage node.
Raw
retrieve_from_object_server.md

Retrieve data from object server

swiftqa@swiftqa01:~$ curl -I -XHEAD "http://192.168.11.11:6000/d41/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
HTTP/1.1 404 Not Found
Content-Type: text/html; charset=UTF-8
Content-Length: 0
Date: Fri, 26 Jun 2015 09:12:51 GMT

swiftqa@swiftqa01:~$ curl -I -XHEAD "http://192.168.11.11:6000/d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" -H "X-Backend-Storage-Policy-Index: 3"
HTTP/1.1 404 Not Found
Content-Type: text/html; charset=UTF-8
Content-Length: 0
Date: Fri, 26 Jun 2015 09:13:04 GMT

Check local disks (2015/06/26)

swiftqa@swiftqa01:~$ ls -al /srv/node/d43/objects-3/2170/972/87a0767a119155053615399f030dd972
total 28172
drwxr-xr-x 2 swift swift       44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#1.data
swiftqa@swiftqa01:~$ ls -al /srv/node/d43/objects-3/2170/972/87a0767a119155053615399f030dd972
total 28172
drwxr-xr-x 2 swift swift       44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#1.data

Check local disks on each nodes (2015/06/27)

ssh swiftqa05 "ls -lah ${DEVICE:-/srv/node*}/d30/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa01 "ls -lah ${DEVICE:-/srv/node*}/d43/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa04 "ls -lah ${DEVICE:-/srv/node*}/d54/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa03 "ls -lah ${DEVICE:-/srv/node*}/d19/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa02 "ls -lah ${DEVICE:-/srv/node*}/d5/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa05 "ls -lah ${DEVICE:-/srv/node*}/d31/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa03 "ls -lah ${DEVICE:-/srv/node*}/d17/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa04 "ls -lah ${DEVICE:-/srv/node*}/d53/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa02 "ls -lah ${DEVICE:-/srv/node*}/d6/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa01 "ls -lah ${DEVICE:-/srv/node*}/d41/objects-3/2170/972/87a0767a119155053615399f030dd972"
ssh swiftqa05 "ls -lah ${DEVICE:-/srv/node*}/d29/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa03 "ls -lah ${DEVICE:-/srv/node*}/d18/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa02 "ls -lah ${DEVICE:-/srv/node*}/d4/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa01 "ls -lah ${DEVICE:-/srv/node*}/d40/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa04 "ls -lah ${DEVICE:-/srv/node*}/d52/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa03 "ls -lah ${DEVICE:-/srv/node*}/d16/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa05 "ls -lah ${DEVICE:-/srv/node*}/d28/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa01 "ls -lah ${DEVICE:-/srv/node*}/d42/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa04 "ls -lah ${DEVICE:-/srv/node*}/d55/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]
ssh swiftqa02 "ls -lah ${DEVICE:-/srv/node*}/d7/objects-3/2170/972/87a0767a119155053615399f030dd972" # [Handoff]

## Result
total 28M
drwxr-xr-x. 2 swift swift  79 Jun 25 12:55 .
drwxr-xr-x. 3 swift swift  53 Jun 25 12:55 ..
-rw-------. 1 swift swift 28M Jun 25 12:55 1435236883.13246#0.data
-rw-r--r--. 1 swift swift   0 Jun 25 12:55 1435236883.13246.durable

ls: cannot access /srv/node*/d43/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

total 28M
drwxr-xr-x 2 swift swift  79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift  53 Jun 25 12:55 ..
-rw------- 1 swift swift 28M Jun 25 12:55 1435236883.13246#2.data
-rw-r--r-- 1 swift swift   0 Jun 25 12:55 1435236883.13246.durable

ssh: connect to host 192.168.11.13 port 22: No route to host

ls: cannot access /srv/node*/d5/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

total 28M
drwxr-xr-x. 2 swift swift  79 Jun 25 12:55 .
drwxr-xr-x. 3 swift swift  53 Jun 25 12:55 ..
-rw-------. 1 swift swift 28M Jun 25 12:55 1435236883.13246#5.data
-rw-r--r--. 1 swift swift   0 Jun 25 12:55 1435236883.13246.durable

ssh: connect to host 192.168.11.13 port 22: No route to host

total 28M
drwxr-xr-x 2 swift swift  79 Jun 25 12:55 .
drwxr-xr-x 3 swift swift  53 Jun 25 12:55 ..
-rw------- 1 swift swift 28M Jun 25 12:55 1435236883.13246#7.data
-rw-r--r-- 1 swift swift   0 Jun 25 12:55 1435236883.13246.durable

ls: cannot access /srv/node*/d6/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

total 28M
drwxr-xr-x 2 swift swift  44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift  53 Jun 25 12:55 ..
-rw------- 1 swift swift 28M Jun 25 12:55 1435236883.13246#9.data

ls: cannot access /srv/node*/d29/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ssh: connect to host 192.168.11.13 port 22: No route to host

ls: cannot access /srv/node*/d4/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ls: cannot access /srv/node*/d40/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ls: cannot access /srv/node*/d52/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ssh: connect to host 192.168.11.13 port 22: No route to host

ls: cannot access /srv/node*/d28/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ls: cannot access /srv/node*/d42/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ls: cannot access /srv/node*/d55/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

ls: cannot access /srv/node*/d7/objects-3/2170/972/87a0767a119155053615399f030dd972: No such file or directory

Check log to find why /d43/objects-3/2170/972/87a0767a119155053615399f030dd972 is gone.

swiftqa@swiftqa01:/var/log/swift$ ls -al all.log.?.gz
-rw-r----- 1 syslog adm 48189290 Jun 24 06:51 all.log.4.gz
-rw-r----- 1 syslog adm 78201657 Jun 23 06:47 all.log.5.gz
-rw-r----- 1 syslog adm 54004426 Jun 22 06:44 all.log.6.gz
-rw-r----- 1 syslog adm 50335557 Jun 21 06:42 all.log.7.gz
-rw-r----- 1 syslog adm 49161259 Jun 20 06:47 all.log.8.gz
-rw-r----- 1 syslog adm 56967885 Jun 19 06:46 all.log.9.gz
swiftqa@swiftqa01:/var/log/swift$ ls -al all.log.?
-rw-r----- 1 syslog adm 133140083 Jun 27 06:29 all.log.1
-rw-r----- 1 syslog adm 255006237 Jun 26 06:48 all.log.2
-rw-r----- 1 syslog adm 423811402 Jun 25 06:43 all.log.3

swiftqa@swiftqa01:/var/log/swift$ cat all.log.?|grep d43/2170
Jun 26 02:36:22 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:02:36:22 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx999d11acc7c34e0ba4e17-00558cbaa6" "proxy-server 11579" 0.0562 "-" 11549 3
Jun 26 04:44:13 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:04:44:13 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-miniscule-readonly-0625_035327_000086_ec64-isolation/objects_000787" 200 153 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-miniscule-readonly-0625_035327_000086_ec64-isolation/objects_000787" "txb23e5b29d1294ff5b4d6d-00558cd89c" "proxy-server 11580" 0.3124 "-" 11548 3
Jun 26 04:53:53 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:04:53:53 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-small-readonly-0625_035327_000086_ec64-isolation/objects_000287" 200 243260 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-small-readonly-0625_035327_000086_ec64-isolation/objects_000287" "tx8841add4db8d4be9b85d6-00558cdae1" "proxy-server 11576" 0.0459 "-" 11548 3
Jun 26 05:35:08 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:05:35:08 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" 404 - "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-large-mixed-0625_035327_000052_ec64-isolation/objects_000066" "tx012cc908873b464697c31-00558ce48c" "proxy-server 24549" 0.1737 "-" 24505 3
Jun 26 06:02:38 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:06:02:38 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-miniscule-mixed-0625_035327_000025_ec64-isolation/objects_000126" 200 228 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-miniscule-mixed-0625_035327_000025_ec64-isolation/objects_000126" "tx401f2c6ace6142a6baf92-00558ceafe" "proxy-server 24554" 0.1897 "-" 24510 3
Jun 26 06:07:24 localhost.localdomain object-server: 192.168.11.11 - - [26/Jun/2015:06:07:24 +0000] "GET /d43/2170/AUTH_test/ssbench-ectest-miniscule-readonly-0625_035327_000086_ec64-isolation/objects_000787" 200 153 "GET http://192.168.11.11/v1/AUTH_test/ssbench-ectest-miniscule-readonly-0625_035327_000086_ec64-isolation/objects_000787" "txcaf9af1696d244829cd49-00558cec1c" "proxy-server 24549" 0.1371 "-" 24512 3
Jun 24 15:43:09 localhost.localdomain object-server: 192.168.12.12 - - [24/Jun/2015:15:43:09 +0000] "SSYNC /d43/2170" 200 - "-" "-" "-" 0.0003 "-" 26554 3
Jun 24 15:43:24 localhost.localdomain object-server: 192.168.12.12/d43/2170 SSYNC LOCK TIMEOUT: 15 seconds: /srv/node/d43/.lock
Jun 24 15:43:24 localhost.localdomain object-server: 192.168.12.12 - - [24/Jun/2015:15:43:24 +0000] "REPLICATE /d43/2170/19a-1e5-25e-26d-278-50d-5de-5f2-64a-6ff-7fb-a43-ac3-aea-b0f-b90-c8e-d68-dd1-e0d-ebc-f1c" 200 50276 "-" "-" "obj-reconstructor 6184" 0.2585 "-" 26552 3

### All disks was unmounted.
swiftqa@swiftqa02:/var/log/swift$ cat all.log.?|grep d43/2170    
swiftqa@swiftqa02:/var/log/swift$ 

### swiftqa03 offline

swiftqa@swiftqa04:/var/log/swift$ cat all.log.?|grep d43/2170
swiftqa@swiftqa04:/var/log/swift$ 

[swiftqa@swiftqa05 swift]$ cat all.log.?|grep d43/2170
[swiftqa@swiftqa05 swift]$
Raw
run_reconstructor_in_script_mode.md

local drives

swiftqa@swiftqa01:/var/log/swift$ ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
total 28172
drwxr-xr-x 2 swift swift       44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#9.data

Run reconstructor in script mode (rebuild #1 in d43)

root@swiftqa04:/var/log/swift$ swift-object-reconstructor /etc/swift/object-server/2.conf -o -v -p 2170

check local drives (d43)

swiftqa@swiftqa01:/var/log/swift$ date; ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
Wed Jul  1 08:27:21 UTC 2015
/srv/node/d41/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       44 Jun 25 12:55 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jun 25 12:55 1435236883.13246#9.data

/srv/node/d43/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jul  1 08:23 .
drwxr-xr-x 3 swift swift       53 Jul  1 08:23 ..
-rw------- 1 swift swift 28841598 Jul  1 08:23 1435236883.13246#1.data
-rw-r--r-- 1 swift swift        0 Jul  1 08:23 1435236883.13246.durable

Run reconstructor in script mode (rebuild #9 in d41)

root@swiftqa05:/var/log/swift$ swift-object-reconstructor /etc/swift/object-server/2.conf -o -v -d30 -p 2170

check local drives (d41)

swiftqa@swiftqa01:/var/log/swift$ date; ls -al /srv/node*/d*/objects-3/2170/972/87a0767a119155053615399f030dd972
Wed Jul  1 08:29:15 UTC 2015
/srv/node/d41/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jul  1 08:29 .
drwxr-xr-x 3 swift swift       53 Jun 25 12:55 ..
-rw------- 1 swift swift 28841598 Jul  1 08:29 1435236883.13246#9.data
-rw-r--r-- 1 swift swift        0 Jul  1 08:29 1435236883.13246.durable

/srv/node/d43/objects-3/2170/972/87a0767a119155053615399f030dd972:
total 28172
drwxr-xr-x 2 swift swift       79 Jul  1 08:23 .
drwxr-xr-x 3 swift swift       53 Jul  1 08:23 ..
-rw------- 1 swift swift 28841598 Jul  1 08:23 1435236883.13246#1.data
-rw-r--r-- 1 swift swift        0 Jul  1 08:23 1435236883.13246.durable
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment