benchmarking different xattr sizes on XFS on a spinning disk

bench-xattr.py

#!/usr/bin/env python

import benchmark
import cPickle as pickle
import hashlib
import os
import xattr

PILE_O_METADATA = pickle.dumps(dict(
    ("attribute%d" % i, hashlib.sha512("thingy %d" % i).hexdigest())
    for i in range(200)))

BASEDIR = '/mnt/sdb1/xattr-test'
METADATA_KEY = 'user.swift.metadata'


def drop_caches():
    if os.environ.get('DROP_CACHES') == '0':
        return
    with open("/proc/sys/vm/drop_caches", 'w') as dc:
        dc.write("3")  # drop everything


def read_metadata(fd):
    """
    Helper function to read the pickled metadata from an object file.

    :param fd: file descriptor or filename to load the metadata from

    :returns: dictionary of metadata

    Stripped out all the error handling so it fit in this benchmark script.
    """
    metadata = ''
    key = 0
    try:
        while True:
            metadata += xattr.getxattr(fd, '%s%s' % (METADATA_KEY,
                                                        (key or '')))
            key += 1
    except IOError:
        pass
    return metadata # pickle.loads(metadata)


def write_metadata(fd, metadata, xattr_size=4096):
    """
    Helper function to write pickled metadata for an object file.

    :param fd: file descriptor or filename to write the metadata
    :param metadata: metadata to write

    Stripped out all the error handling so it fit in this benchmark script. Also moved the pickling out.

    """
    metastr = metadata # was pickle.dumps(metadata, 2)
    key = 0
    while metastr:
        xattr.setxattr(fd, '%s%s' % (METADATA_KEY, key or ''),
                       metastr[:xattr_size])
        metastr = metastr[xattr_size:]
        key += 1


class BenchmarkXattrSizes(benchmark.Benchmark):
    def fname(self, xattr_size):
        return os.path.join(BASEDIR, 't%d' % xattr_size)

    def write_it(self, xattr_size):
        with open(self.fname(xattr_size), 'w') as fh:
            fh.write('test with %d' % xattr_size)
            write_metadata(fh.fileno(), PILE_O_METADATA,
                           xattr_size=xattr_size)


    def setUp(self):
        self.write_it(254)  # current on master
        self.write_it(1024)
        self.write_it(2048)
        self.write_it(4096)
        self.write_it(8192)
        self.write_it(16384)
        self.write_it(32768)
        print "metadata length is %d bytes" % len(PILE_O_METADATA)

    def eachSetUp(self):
        drop_caches()

    def test_254(self):
        read_metadata(self.fname(254))

    def test_1024(self):
        read_metadata(self.fname(1024))

    def test_2048(self):
        read_metadata(self.fname(2048))

    def test_4096(self):
        read_metadata(self.fname(4096))

    def test_8192(self):
        read_metadata(self.fname(8192))

    def test_16384(self):
        read_metadata(self.fname(16384))

    def test_32768(self):
        read_metadata(self.fname(32768))


if __name__ == '__main__':
    benchmark.main(each=2500)

results with buffer cache

(bench-venv)root@swift-test-02:~# DROP_CACHES=0 python bench-xattr.py 
metadata length is 31590 bytes

Benchmark Report
================

BenchmarkXattrSizes
-------------------

 name | rank | runs |      mean |        sd | timesBaseline
------|------|------|-----------|-----------|--------------
32768 |    1 | 2500 | 4.424e-05 | 3.192e-06 |           1.0
16384 |    2 | 2500 | 5.901e-05 | 4.004e-06 | 1.33393267535
 8192 |    3 | 2500 | 8.495e-05 | 4.782e-06 | 1.92041686609
 4096 |    4 | 2500 | 0.0001466 | 7.201e-05 | 3.31461144096
 2048 |    5 | 2500 | 0.0002551 | 1.173e-05 | 5.76624798962
 1024 |    6 | 2500 | 0.0004584 |  4.08e-05 | 10.3637014328
  254 |    7 | 2500 |  0.001729 | 1.638e-05 | 39.0832287719

Each of the above 17500 runs were run in random, non-consecutive order by
`benchmark` v0.1.5 (http://jspi.es/benchmark) with Python 2.7.3
Linux-3.2.0-23-generic-x86_64 on 2014-12-05 22:09:23.

results without buffer cache (dropping cache each time)

(bench-venv)root@swift-test-02:~# DROP_CACHES=1 python bench-xattr.py 
metadata length is 31590 bytes

Benchmark Report
================

BenchmarkXattrSizes
-------------------

 name | rank | runs |      mean |        sd | timesBaseline
------|------|------|-----------|-----------|--------------
32768 |    1 | 2500 | 0.0001195 |  3.75e-05 |           1.0
16384 |    2 | 2500 | 0.0001348 | 1.869e-05 | 1.12809122912
 8192 |    3 | 2500 | 0.0001604 | 2.708e-05 | 1.34210998858
 4096 |    4 | 2500 | 0.0002326 | 0.0004816 | 1.94623473988
 2048 |    5 | 2500 | 0.0003414 | 0.0001409 | 2.85674781189
 1024 |    6 | 2500 | 0.0005457 | 0.0001741 | 4.56648611635
  254 |    7 | 2500 |  0.001848 |  0.001663 | 15.4616067887

Each of the above 17500 runs were run in random, non-consecutive order by
`benchmark` v0.1.5 (http://jspi.es/benchmark) with Python 2.7.3
Linux-3.2.0-23-generic-x86_64 on 2014-12-05 22:10:04.