On all six GlusterFS server/client nodes:
### Install iozone
[root@core-n1 ~]# yum install http://pkgs.repoforge.org/iozone/iozone-3.408-1.el6.rf.x86_64.rpm -y
### Mount the volume
[root@core-n1 ~]# gluster peer status
Number of Peers: 5
Hostname: core-n2.storage-s0.example.vpn
Uuid: de49dd15-c05f-4ef8-b1e0-c29c59623b5
State: Peer in Cluster (Connected)
Hostname: core-n3.storage-s0.example.vpn
Uuid: 40ac9d8a-ce05-4964-b8f3-802e7ccd2ff
State: Peer in Cluster (Connected)
Hostname: core-n4.storage-s0.example.vpn
Uuid: 26201442-b264-4078-a2d3-ff61a266f93
State: Peer in Cluster (Connected)
Hostname: core-n5.storage-s0.example.vpn
Uuid: 7a637a4b-b9a9-4dd5-80b4-78474e9e33b
State: Peer in Cluster (Connected)
Hostname: core-n6.storage-s0.example.vpn
Uuid: 2cc8c574-a2e7-4819-a175-e7218c8606d
State: Peer in Cluster (Connected)
[root@core-n1 ~]# gluster volume info
Volume Name: benchmark-vol-n1
Type: Distributed-Replicate
Volume ID: 3da7699f-165f-4357-b20c-668bc109d97
Status: Started
Number of Bricks: 4 x 3 = 12
Transport-type: tcp
Bricks:
Brick1: core-n1.storage-s0.example.vpn:/export/brick01/data
Brick2: core-n2.storage-s0.example.vpn:/export/brick01/data
Brick3: core-n3.storage-s0.example.vpn:/export/brick01/data
Brick4: core-n4.storage-s0.example.vpn:/export/brick01/data
Brick5: core-n5.storage-s0.example.vpn:/export/brick01/data
Brick6: core-n6.storage-s0.example.vpn:/export/brick01/data
Brick7: core-n1.storage-s0.example.vpn:/export/brick02/data
Brick8: core-n2.storage-s0.example.vpn:/export/brick02/data
Brick9: core-n3.storage-s0.example.vpn:/export/brick02/data
Brick10: core-n4.storage-s0.example.vpn:/export/brick02/data
Brick11: core-n5.storage-s0.example.vpn:/export/brick02/data
Brick12: core-n6.storage-s0.example.vpn:/export/brick02/data
Options Reconfigured:
cluster.server-quorum-type: server
performance.io-thread-count: 5
auth.allow: 10.30.*
cluster.server-quorum-ratio: 51%
[root@core-n1 ~]# glusterfs \
-s core-n1.storage-s0.example.vpn \
-s core-n2.storage-s0.example.vpn \
-s core-n3.storage-s0.example.vpn \
-s core-n4.storage-s0.example.vpn \
-s core-n5.storage-s0.example.vpn \
-s core-n6.storage-s0.example.vpn \
--volfile-id=/etc/glusterfs/glusterd.vol \
/mnt
### Inspect the mounted volume and its local bricks
[root@core-n1 ~]# mount|grep /mnt
core-n1.storage-s0.example.vpn:benchmark-vol-n1 on /mnt type fuse.glusterfs (rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)
[root@core-n1 ~]#
[root@core-n1 ~]# df -h /export/brick01 /export/brick02 /mnt
Filesystem Size Used Avail Use% Mounted on
/dev/sdb 2.8T 6.9M 2.8T 1% /export/brick01
/dev/sdc 2.8T 17M 2.8T 1% /export/brick02
core-n1.storage-s0.example.vpn:benchmark-vol-n1 11T 2.3G 11T 1% /mnt
[root@core-n1 ~]#
### Create one directory per host for benchmark data
[root@core-n1 ~]# mkdir -p /mnt/tmp/$(hostname -s)
[root@core-n1 ~]# ls /mnt/tmp
core-n1 core-n2 core-n3 core-n4 core-n5 core-n6
[root@core-n1 ~]#
### Assess the amount of RAM installed. These servers contain 256GB of RAM.
[root@core-n1 ~]# free -g
total used free shared buffers cached
Mem: 251 1 249 0 0 0
-/+ buffers/cache: 1 249
Swap: 3 0 3
[root@core-n1 ~]#
### A value of 3 times the total RAM installed in each server
### should be used as the size of the iozone test file.
### For these servers, the test file size would be 756GB, if we
### ran iozone with one thread. Using a test file much larger
### than the amount of RAM installed provides greater confidence
### that the measurement will reflect performance beyond the
### effects of cache.
### Each of these servers also contains 12 physical cores.
### For this reason, we will be running 12 threads with iozone.
### As we will be running with 12 threads, we should probably
### divide the total test file size by 12. Therefore, we will
### configure iozone to create 64GB files.
### Get comfortable with the iozone documentation.
### http://www.iozone.org/docs/IOzone_msword_98.pdf
#### Purge caches
sync; echo 3 > /proc/sys/vm/drop_caches
### Allow iozone to run uninterrupted for a very long time, possibly days.
[root@core-n1 ~]# cd /mnt/tmp/$(hostname -s)
[root@core-n1 core-n1]# iozone -c -e -i 0 -i 1 -+n -s 64g -t 12 -R -b /tmp/iozone-test-1a_$(hostname -s).xls
### You may wish to run the random read-write test as well (add '-i 2').
### If you do, be prepared to wait a long, long, long while.