aliyun_oss_sync.py

#!/usr/bin/env python

import configargparse
import hashlib
import os
import oss2

def _local_etag(data):
    return hashlib.md5(data).hexdigest().upper()

def _remote_etag(key):
    try:
        return bucket.get_object_meta(key).etag
    except:
        return None

def _upload_file(key, data):
    print "Uploading ...",
    try:
        bucket.put_object(key, data)
        print "Success!"
    except oss2.exceptions as err:
        print err

def _download_file(key):
    print "Downloading ...",
    try:
        file_obj = bucket.get_object(key)
        with open(key, 'wb') as f:
            f.write(file_obj.read())
        print "Success!"
    except oss2.exceptions as err:
        print err

def _uri_encode(f, pfix):
    f_list = f.split(os.path.sep)
    uri = '/'.join(f_list[1:])
    return "%s%s" % (pfix, uri)

def upload_files(path):
    os.chdir(path)
    path = os.curdir
    for dir_path, dir_names, file_names in os.walk(path):
        for file_name in file_names:
            f = os.path.join(dir_path, file_name)
            key = _uri_encode(f, bucket_prefix)
            with open(f, 'rb') as f_obj:
                data = f_obj.read()
                remote_etag = _remote_etag(key)
                local_etag = _local_etag(data)
                if remote_etag is None:
                    print "File: %s doesn't exist. -> " % f,
                    _upload_file(key, data)
                elif remote_etag != local_etag:
                    print "File: %s exists, but etag '%s' != '%s'. -> " % (f, local_etag, remote_etag),
                    _upload_file(key, data)
                else:
                    print "File: %s already exists -> skip" % f

def download_files(path):
    os.chdir(path)
    marker = ''
    obj_list = []
    while True:
        list_obj = bucket.list_objects(bucket_prefix, '', marker)
        marker = list_obj.next_marker
        obj_list += list_obj.object_list
        if not list_obj.is_truncated:
            break
    for obj in obj_list:
        key = obj.key
        if key[-1] == '/':
            continue
        elif os.path.exists(key) is True:
            remote_etag = obj.etag
            with open(key, 'rb') as data:
                local_etag = _local_etag(data.read())
                if local_etag == remote_etag:
                    print "File %s exists -> skip" % key
                else:
                    print "File %s exists, but etag %s != %s -> " % (key, local_etag, remote_etag),
                    _download_file(key)
        else:
            dir_name = os.path.dirname(key)
            if dir_name and os.path.exists(dir_name) is False:
                os.makedirs(dir_name)
            print "File %s doesn't exist -> " % key,
            _download_file(key)
            
if __name__ == '__main__':
    p = configargparse.ArgParser()
    p.add('action', choices=['download', 'upload'], help='action to perform')
    p.add('-b', '--bucket', required=True, help='bucket name', metavar='')
    p.add('-bp', '--bucket-prefix', default='', help='bucket prefix', metavar='')
    p.add('-e', '--endpoint', required=True, help='api endpoint', metavar='')
    p.add('-p', '--path', required=True, help='local path', metavar='')
    options = p.parse_args()

    access_key = 'ACCESS_KEY'
    secret_key = 'SECRET_KEY'
    bucket_prefix = options.bucket_prefix

    try:
        auth = oss2.Auth(access_key, secret_key)
        bucket = oss2.Bucket(auth, options.endpoint, options.bucket)

        if options.action == 'download':
            download_files(options.path)
        elif options.action == 'upload':
            upload_files(options.path)
    except Exception as err:
        print err