Skip to content

Instantly share code, notes, and snippets.

@zxygentoo
Created June 15, 2017 13:24
Show Gist options
  • Save zxygentoo/4edc993b6b40056840fcafe8e0b23ce3 to your computer and use it in GitHub Desktop.
Save zxygentoo/4edc993b6b40056840fcafe8e0b23ce3 to your computer and use it in GitHub Desktop.
This script sync all files of a given dir to a given aliyun oss bucket and keep all the sub-dir as part of the keys.
#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
This script sync all files of a given dir to a given aliyun oss bucket
and keep all the sub-dir as part of the keys.
"""
from __future__ import print_function
from os import path, walk
from sys import stdout
from oss2 import Auth, Bucket
def sync_dir_to_bucket(dir, bucket):
"""Get all filenames in dir, convert to oss keys, then put to bucket."""
absdir = path.abspath(dir)
names = list_dir_filenames(absdir)
keys = filenames_to_oss_keys(absdir, names)
print('Start syncing files in [ %s ]' % absdir)
for counter, (key, name) in enumerate(zip(keys, names)):
bucket.put_object_from_file(
key, name,
progress_callback=display_upload_progress
)
print(
'[%d] %s ==> %s' % (
counter + 1,
short_filename(absdir, name),
full_oss_key(bucket, key)
)
)
def list_dir_filenames(dir):
"""Return all full filenames with path in dir as a list."""
return [
path.join(root, f)
for root, _, files in walk(dir)
for f in files
]
def filenames_to_oss_keys(dir, filenames):
"""Convert list filenames to a list of oss keys."""
return [
f.replace(path.join(dir, ''), '')
for f in filenames
]
def get_bucket(key, secret, endpoint, bucket):
"""Return an oss2 bucket."""
return Bucket(Auth(key, secret), endpoint, bucket)
def full_oss_key(bucket, key):
"""Generate a nice-looking `oss://bucket/key` string."""
return 'oss://%s/%s' % (bucket.bucket_name, key)
def short_filename(dir, filename):
"""Remove base dir from filename make it look nicer."""
return filename.replace(path.join(dir, ''), '')
def display_upload_progress(consumed_bytes, total_bytes):
"""Display file upload progress."""
if total_bytes:
print('\r{0}% '.format(
int(100 * (float(consumed_bytes)) / (float(total_bytes)))
), end='')
stdout.flush()
if __name__ == "__main__":
KEY = 'YOUR-KEY'
SECRET = 'YOUR-SECRET'
ENDPOINT = 'YOUR-ENDPOINT'
BUCKET = 'YOUR-BUCKET'
DIR = 'DIR-TO-SYNC'
sync_dir_to_bucket(DIR, get_bucket(KEY, SECRET, ENDPOINT, BUCKET))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment