upyun podcast uploader with filename-as-cache

upyun_podcast_uploader.py

import os
import upyun
from progressbar import *

# http://stackoverflow.com/questions/1192978/python-get-relative-path-of-all-files-and-subfolders-in-a-directory
myFolder = "/podcast/pod/podcasts/"
os.chdir(myFolder)
fileSet = set()

# walk through
for root, dirs, files in os.walk(myFolder):
    for fileName in files:
        fileSet.add(os.path.join(root[len(myFolder):], fileName))

try:
    bucket = os.environ['UPYUN_BUCKET']
    username = os.environ['UPYUN_USERNAME']
    password = os.environ['UPYUN_PASSWORD']
except:
    bucket, username, password = ('ahxxm', 'ahxxm', 'some_strong_password')

up = upyun.UpYun(bucket, username, password, timeout=30, endpoint=upyun.ED_AUTO, chunksize=4096)

raw_all_folders = up.getlist('/')
all_folders = []
for item in raw_all_folders:
    if item['size'] == '0':
        all_folders.append(item)

# 'folder_name': [file list]
all_files = dict()


class ProgressBarHandler(object):
    def __init__(self, totalsize, params):
        widgets = [params, Percentage(), ' ',
                   Bar(marker='=', left='[', right=']'), ' ',
                   ETA(), ' ', FileTransferSpeed()]
        self.pbar = ProgressBar(widgets=widgets, maxval=totalsize).start()

    def update(self, readsofar):
        self.pbar.update(readsofar)

    def finish(self):
        self.pbar.finish()


def upload_file(filepath):
    with open(filepath, 'rb') as f:
        # Check cache
        folder, filename = filepath.split('/')
        print filepath
        if folder not in all_folders:
            try:
                all_files[folder] = up.getlist('/' + folder + '/')
                # Hit cache then skip
                current_folder_filelist = all_files[folder]
                for file_item in current_folder_filelist:
                    if file_item['name'] == filename:
                        return
            except:
                # new folder
                pass

        # upload
        try:
            res = up.put(filepath, f, handler=ProgressBarHandler, params='Uploading ')
        except:
            pass


for filepath in fileSet:
    upload_file(filepath)

then crontab -e, add 2 lines:

45 * * * * /bin/python /podcast/pod/pod.py -u

45 10,20 * * * /bin/python /podcast/pod/sync.py

.. however upyun charges "only for free when bandwidth exceeds your storage", so there's a new script that uploads the folder to Google Drive