Skip to content

Instantly share code, notes, and snippets.

@tenstormavi
Created June 11, 2019 12:26
Show Gist options
  • Save tenstormavi/75ceee8dcdcee8787e203a11e4255434 to your computer and use it in GitHub Desktop.
Save tenstormavi/75ceee8dcdcee8787e203a11e4255434 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python3
# -*- mode: python -*-
import sys
import hashlib
def calculate_multipart_etag(source_path, chunk_size, expected=None):
md5s = []
with open(source_path,'rb') as fp:
while True:
data = fp.read(chunk_size)
if not data:
break
md5s.append(hashlib.md5(data))
if len(md5s) > 1:
digests = b"".join(m.digest() for m in md5s)
new_md5 = hashlib.md5(digests)
new_etag = '"%s-%s"' % (new_md5.hexdigest(),len(md5s))
elif len(md5s) == 1:
# file smaller than chunk size
new_etag = '"%s"' % md5s[0].hexdigest()
else:
new_etag = '""'
if expected:
if not expected==new_etag:
raise ValueError('new etag %s does not match expected %s' % (new_etag,expected))
return new_etag
if __name__ == '__main__':
source_path = sys.argv[1]
chunk_size = sys.argv[2]
chunk_size = int(chunk_size) * 1024 * 1024
try:
expected = '"%s"' % (sys.argv[3])
except Exception as e:
expected = None
print(calculate_multipart_etag(source_path,chunk_size,expected))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment