network tuning

# Jumbo frames are enabled on the compute nodes and switches in the cluster 
# so we are able to utilize more bandwidth at the VM level by increasing
# the ethernet frame payload size to 9000 bytes
ifconfig eth0 mtu 9000

# Increase the transmit queue length
ifconfig eth0 txqueuelen 10000

# Increase the receive and send buffers to 16MB
net.core.rmem_max=16777216
net.core.wmem_max=16777216

# Set the minimum, default, and maximum buffer sizes
net.ipv4.tcp_rmem="4096 87380 16777216"
net.ipv4.tcp_wmem="4096 65536 16777216"

# Increase the number of packets that will get queued if they are being
# received faster than the kernel can process them
net.core.netdev_max_backlog=30000

# We want to prevent the VM from paging, see below
vm.swappiness=0

# Since swap is not enabled we want to make sure 64MB of RAM is reserved to
# prevent Out of memory scenarios
# http://en.wikipedia.org/wiki/Out_of_memory
vm.min_free_kbytes=65536

results

Using default CentOS/RHEL 6.4 settings:

Recv   Send    Send                          Utilization     
Socket Socket  Message  Elapsed              Send     Recv  
Size   Size    Size     Time     Throughput  local    remote
bytes  bytes   bytes    secs.    10^6bits/s  % S      % S    

 87380  16384  16384    300.00     5257.54   6.77     8.86
 87380  16384  16384    300.00     5288.45   6.59     8.96
 87380  16384  16384    300.00     5271.40   6.41     8.57
 87380  16384  16384    300.00     5125.45   6.26     8.67
 87380  16384  16384    300.00     4982.35   6.52     8.48

With tuning:

Recv   Send    Send                          Utilization    
Socket Socket  Message  Elapsed              Send     Recv  
Size   Size    Size     Time     Throughput  local    remote
bytes  bytes   bytes    secs.    10^6bits/s  % S      % S    

 87380  65536  65536    300.01     6821.60   8.28     12.47
 87380  65536  65536    300.01     7091.14   8.61     12.57
 87380  65536  65536    300.02     6820.71   8.33     12.33
 87380  65536  65536    300.01     7073.85   8.79     12.48
 87380  65536  65536    300.01     7039.33   8.46     12.46

run netperf

# Run 'netserver' on one host

# On another host, provide the IP address of the host running netserver
netperf -H <netperf_host> -p 12865 -P1 -l 300 -C -c -f m -t TCP_STREAM

# Perform five runs