copy-to-rados.py

#!/usr/bin/env python

try:
    import rados
    import sys
    import hashlib
    import os
except ImportError:
    raise ImportError('unable to import modules')
    sys.stderr.write("unable to import modules")
    exit(1)

def get_filepaths(directory):
    """
    This function will generate the file names in a directory
    tree by walking the tree either top-down or bottom-up. For each
    directory in the tree rooted at directory top (including top itself),
    it yields a 3-tuple (dirpath, dirnames, filenames).
    """
    file_paths = []  # List which will store all of the full filepaths.

    # Walk the tree.
    for root, directories, files in os.walk(directory):
        for filename in files:
            # Join the two strings in order to form the full filepath.
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)  # Add it to the list.

    return file_paths  # Self-explanatory.

# Run the above function and store its results in a variable.

def copy_to_ceph(afile, key, ioctx_handler, blocksize=65536):
    print afile
    print key
    length = os.path.getsize(afile)
    f = open(afile, 'r')
    buf = f.read(blocksize)
    h = hashlib.md5()
    offset = 0
    while len(buf) > 0:
        h.update(buf)
        ioctx.write(key, buf, offset)
        offset += len(buf)
        buf = f.read(blocksize)

    ioctx.set_xattr(key, "MD5", h.hexdigest())

    print "finished copying file %s" % key
    return True

if __name__ == "__main__":
  cluster = rados.Rados(conffile=ceph_conf_file,
                          conf=dict(keyring=ceph_keyring))

  cluster.connect()
  ioctx = cluster.open_ioctx(ceph_pool)

  # do somehting with files
  for f in get_filepaths("/tmp")
    copy_to_ceph(f, f, ioctx, 1048576)
  
  ioctx.close()
  cluster.shutdown()