Skip to content

Instantly share code, notes, and snippets.

@hervenivon
Created November 18, 2020 21:31
Show Gist options
  • Save hervenivon/bb5133725b4ce5eedcb0a5c46d21d961 to your computer and use it in GitHub Desktop.
Save hervenivon/bb5133725b4ce5eedcb0a5c46d21d961 to your computer and use it in GitHub Desktop.
Upload large files in multi part
import boto3
import os
import sys
import threading
from boto3.s3.transfer import TransferConfig
s3_client = boto3.client('s3')
# S3 transfer documentation: https://boto3.amazonaws.com/v1/documentation/api/1.9.156/reference/customizations/s3.html?highlight=transferconfig#boto3.s3.transfer.TransferConfig
# From boto3.s3.transfer: source code https://boto3.amazonaws.com/v1/documentation/api/latest/_modules/boto3/s3/transfer.html
class ProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage))
sys.stdout.flush()
def uploadFileS3(filepath, bucket, keypath=''):
config = TransferConfig(multipart_threshold=1024*25,
max_concurrency=10,
multipart_chunksize=1024*25,
use_threads=True)
head, tail = os.path.split(filepath)
key = keypath + tail
# https://boto3.amazonaws.com/v1/documentation/api/1.9.185/guide/s3-uploading-files.html
s3_client.upload_file(filepath, bucket, key,
Config = config,
Callback=ProgressPercentage(filepath)
)
uploadFileS3('/tmp/500mfile', 'nivonh-poc')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment