Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
使用分块上传上传5T大文件
# -*- coding: UTF-8 -*-
import requests
import xmltodict
from xmltodict import OrderedDict
from awsauth import S3Auth
from multiprocessing.dummy import Pool as ThreadPool
host = '192.168.10.201:7480'
access_key = 'user1'
secret_key = 'user1'
bucketname = 'bucket1'
objectname = '30-2MB'
count_of_10MB = 3 # 524288 5T
#load 10MB data
with open('10MB', 'rb') as fin:
data = fin.read()
url = 'http://{}/{}/{}?uploads'.format(host,bucketname,objectname)
print "url: {}".format(url)
response = requests.post(url, auth=S3Auth(access_key, secret_key,service_url=host))
UploadId = xmltodict.parse(response.content)['InitiateMultipartUploadResult']['UploadId']
print "UploadId: {}".format(UploadId)
def upload_part(i):
print "current uploading part ( {} / {} )".format(i,count_of_10MB)
url = 'http://{}/{}/{}?partNumber={}&uploadId={}'.format(host,bucketname,objectname,i,UploadId)
response = requests.put(url, auth=S3Auth(access_key, secret_key,service_url=host),data=data)
print "status of part {} : {}".format(i,response.status_code)
pool = ThreadPool(20)
results = pool.map(upload_part, range(1,count_of_10MB+1))
pool.close()
pool.join()
url = 'http://{}/{}/{}?uploadId={}'.format(host,bucketname,objectname,UploadId)
response = requests.get(url, auth=S3Auth(access_key, secret_key,service_url=host))
obj2 = []
for i in xmltodict.parse(response.content)['ListPartsResult']['Part']:
obj2.append({'PartNumber': i['PartNumber'], 'ETag': i['ETag']})
obj = {'CompleteMultipartUpload': OrderedDict((
('Part',obj2),
))}
completeme = xmltodict.unparse(obj,full_document=False)
url = 'http://{}/{}/{}?uploadId={}'.format(host,bucketname,objectname,UploadId)
response = requests.post(url, auth=S3Auth(access_key, secret_key,service_url=host),data=completeme)
@wzyuliyang

This comment has been minimized.

Copy link
Owner Author

wzyuliyang commented Dec 20, 2016

create 10MB part object for 5TB
touch 10MB
truncate -s10M 10MB

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.