Example mp download boto

mpdownload.py

#!/usr/bin/env python

from boto     import connect_s3
from argparse import ArgumentParser
from tempfile import TemporaryFile
from shutil   import copyfileobj
from ssl      import SSLError
from socket   import error as SocketError


def fetch_single_part(object, head_byte, tail_byte, offset, fp):
    part_size = 1 + tail_byte - head_byte
    tempfile = TemporaryFile()
    for _ in range(3):
        tempfile.seek(0, 0)
        try:
            object.get_contents_to_file(
                    tempfile,
                    headers={'Range': 'bytes=%d-%d' % (head_byte, tail_byte)}
                    )
        except (SSLError, SocketError):
            continue

        tempfile.seek(0, 2)
        size = tempfile.tell()
        if size == part_size:
            break
        else:
            print('Invalid size: %d vs %d' % (size, part_size))
            tempfile.truncate(0)
    else:
        raise RuntimeError('Failed to fetch sanely')


    fp.seek(head_byte, 0)
    tempfile.seek(0, 0)
    copyfileobj(tempfile, fp)
    return part_size


def main(argv=None):
    argp = ArgumentParser()
    argp.add_argument(
            '--bucket',
            '-B',
            type=str,
            required=True)
    argp.add_argument(
            '--object',
            '-O',
            type=str,
            required=True)
    argp.add_argument(
            '--output',
            '-o',
            type=str,
            required=True)
    argp.add_argument(
            '--chunk-size',
            '-c',
            type=str,
            default=64 * 1024 ** 2)

    args = argp.parse_args(argv)

    # --
    connection = connect_s3()
    bucket = connection.get_bucket(args.bucket)
    object = bucket.get_key(args.object)

    # --
    total_file_size = object.size
    copied_size = 0

    # 1. Initialize sparse file
    with open(args.output, 'wb') as fp:
        while copied_size != total_file_size:
            print('%s' % (copied_size / float(total_file_size)))
            head = copied_size
            tail = min(total_file_size - 1, copied_size + args.chunk_size)
            print('%5d %d' % (head, tail))
            copied_size += \
                    fetch_single_part(object, head, tail, copied_size, fp)

    return 0


if __name__ == '__main__':
    import sys
    sys.exit(main(sys.argv[1:]))