Skip to content

Instantly share code, notes, and snippets.

@Millnert
Created March 5, 2017 10:24
Show Gist options
  • Save Millnert/8f938162867bb6e752ba2f6f57c44223 to your computer and use it in GitHub Desktop.
Save Millnert/8f938162867bb6e752ba2f6f57c44223 to your computer and use it in GitHub Desktop.
python-rados writes to ec-pool
def put_object(ioctx, diskpath, objprefix):
""" Puts an object from disk, using (objprefix + filename) as key.
Naive code which reads entire object to memory before writing.
"""
bufsize = 2**20 ## 1 MiB
key = objprefix + os.path.basename(diskpath)
offset = 0
sha = hashlib.sha512()
fhd = open(diskpath, 'rb')
while True:
datachunk = fhd.read(bufsize)
if datachunk != "":
sha.update(datachunk)
print("key: %s, chunk: %d, offset: %d" % (key, len(datachunk), offset))
completion = ioctx.aio_write(key, datachunk, offset)
completion.wait_for_safe_and_cb()
offset += bufsize ## usually not true on the last of several
## chunked reads, but doesn't matter then
else:
break
fhd.close()
ioctx.set_xattr(key, 'sha512sum', sha.hexdigest())
def put_object(ioctx, diskpath, objprefix):
""" Puts an object from disk, using (objprefix + filename) as key.
Naive code which reads entire object to memory before writing.
"""
bufsize = 2**20 ## 1 MiB
key = objprefix + os.path.basename(diskpath)
offset = 0
sha = hashlib.sha512()
fhd = open(diskpath, 'rb')
while True:
datachunk = fhd.read(bufsize)
if datachunk != "":
sha.update(datachunk)
print("key: %s, chunk: %d, offset: %d" % (key, len(datachunk), offset))
if offset == 0:
completion = ioctx.aio_write(key, datachunk, offset)
else:
completion = ioctx.aio_append(key, datachunk)
completion.wait_for_safe_and_cb()
offset += bufsize ## usually not true on the last of several
## chunked reads, but doesn't matter then
else:
break
fhd.close()
ioctx.set_xattr(key, 'sha512sum', sha.hexdigest())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment