Skip to content

Instantly share code, notes, and snippets.

@rcrowley
Created November 14, 2013 22:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rcrowley/7475762 to your computer and use it in GitHub Desktop.
Save rcrowley/7475762 to your computer and use it in GitHub Desktop.
Support X-Delete-After headers and fix segmented uploads in Rackspace's Swift Tool, st.
--- st.orig 2013-11-14 21:59:32.014428532 +0000 +++ st 2013-11-14 22:45:48.542834157 +0000
@@ -1455,6 +1455,8 @@
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--ttl', dest='ttl', help='Set a time-to-live for '
+ 'the object via the X-Delete-After header.')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1469,8 +1471,12 @@
else:
fp = open(job['path'], 'rb')
fp.seek(job['segment_start'])
+ headers = {}
+ if options.ttl:
+ headers['x-delete-after'] = options.ttl
conn.put_object(job.get('container', args[0] + '_segments'),
- job['obj'], fp, content_length=job['segment_size'])
+ job['obj'], fp, content_length=job['segment_size'],
+ headers=headers)
if options.verbose and 'log_line' in job:
print_queue.put(job['log_line'])
@@ -1483,6 +1489,8 @@
if obj.startswith('./') or obj.startswith('.\\'):
obj = obj[2:]
put_headers = {'x-object-meta-mtime': str(getmtime(path))}
+ if options.ttl:
+ put_headers['x-delete-after'] = options.ttl
if dir_marker:
if options.changed:
try:
@@ -1521,7 +1529,7 @@
if err.http_status != 404:
raise
if options.segment_size and \
- getsize(path) < options.segment_size:
+ getsize(path) > int(options.segment_size):
full_size = getsize(path)
segment_queue = Queue(10000)
segment_threads = [QueueFunctionThread(segment_queue,
@@ -1551,8 +1559,8 @@
while thread.isAlive():
thread.join(0.01)
new_object_manifest = '%s_segments/%s/%s/%s/' % (
- container, obj, put_headers['x-object-meta-mtime'],
- full_size)
+ container, basename(obj),
+ put_headers['x-object-meta-mtime'], full_size)
if old_manifest == new_object_manifest:
old_manifest = None
put_headers['x-object-manifest'] = new_object_manifest
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment