md5 generator for OCI object storage multipart upload

md5_multipart_upload.py

#!/usr/bin/env python3
import argparse
import hashlib
import sys
import codecs

def md5(f, count):
    hash_md5 = hashlib.md5()
    eof = False
    for i in range(count * 16):
        chunk = f.read(65536)
        if chunk == b'':
           eof = True
           break
        hash_md5.update(chunk)
    return (hash_md5.digest(), eof)

parser = argparse.ArgumentParser()
parser.add_argument('filename',
                    help='File that will be used to calculate the MD5 sum on')
parser.add_argument('partsize',
                    type=int,
                    help='Size of individual parts in (MiB)')
parser.add_argument('--base64',
                    action='store_true',
                    help='Display in base64 instead of hexadecimal')
parser.add_argument('--verbose',
                    action='store_true',
                    help='Show checksums for each part')
cli_options = parser.parse_args()

try:
    f=open(cli_options.filename, 'rb')
except IOError:
    print ('Cannot open file')
    sys.exit(1)

eof = False
hash_list = []
while eof == False:
    (md5_hash, eof) = md5(f, cli_options.partsize)
    if cli_options.verbose == True:
        print (codecs.encode(md5_hash, 'hex').decode())
    hash_list.append(md5_hash)
f.close()

multipart_hash = hashlib.md5(b''.join(hash_list)).hexdigest()
if cli_options.base64 == True:
    multipart_hash = codecs.encode(codecs.decode(multipart_hash, 'hex'), 'base64').decode().rstrip()

print ('%s-%d' % (multipart_hash, len(hash_list)))

Generate md5 checksum for a local file using the same approach that Oracle Cloud Infrastructure object storage service uses with multipart uploads. The md5 checksum is created for each part, these are concatenated together, and a final md5 is created from the list of md5's.

Use --base64 to show result in base64, to match the format used by OCI.

E.g: md5_multipart_upload.py --base64 <filename>