DropboxSync.py

import shutil
import os
import sys
import pickle
from dropbox.rest import ErrorResponse


STATE_FILE = '.dropbox_state'
REMOTE_ROOT_PATH = '/Apps/my-pythonista-app' # sync root in dropbox where code is


class dropbox_state:
    def __init__(self):
        self.cursor = None
        self.local_files = {}
        self.remote_files = {}
        self.remote_root_path = REMOTE_ROOT_PATH

    # use ignore_path to prevent download of recently uploaded files 
    def execute_delta(self, client, ignore_path=None):
        delta = client.delta(self.cursor)
        self.cursor = delta['cursor']

        remote_root_path = self.remote_root_path.lower()
        if ignore_path:
            ignore_path = ignore_path.lower()

        for entry in delta['entries']:
            path, meta = entry
            if not path.startswith(remote_root_path):
                continue

            # this skips the path if we just uploaded it
            if path != ignore_path:
                if meta != None:
                    if not meta['is_dir']:
                        if path not in self.local_files:
                            print '\n\tNot in local'
                            self.download(client, path)
                        elif meta['rev'] != self.remote_files[path]['rev']:
                            print '\n\tOutdated revision'
                            self.download(client, path)
                # remove file or directory
                else:
                    local_path = self.remote_path_to_local_path(client, path)
                    if os.path.isdir(local_path):
                        print '\n\tRemoving Directory:', local_path
                        shutil.rmtree(local_path, True)
                    elif os.path.isfile(local_path):
                        print '\n\tRemoving File:', local_path

                        for files in (self.local_files, self.remote_files):
                            if path in files:
                                del files[path]
                            else:
                                print '\n\tAlready removed?:', path
                        os.remove(local_path)

        if delta['has_more']:
            self.execute_delta(client, ignore_path=ignore_path)

    def local_path_to_remote_path(self, client, local_path):
        return '/'.join(filter(None, [self.remote_root_path, local_path])).lower()

    def remote_path_to_local_path(self, client, remote_path):
        # obtain relative actual local path
        # remote_path is case insensitive and path of metadata may be case insensitive sometime
        # XXX I found file name and their parent directory in metadata are case sensitive, but not sure ancestors.
        remote_path_names = remote_path[1:].split('/') 
        for i in range(len(remote_path_names)):
            path = '/' + '/'.join(remote_path_names[:i + 1])
            if path not in self.remote_files:
                self.remote_files[path] = client.metadata(path)
        l = []
        for i in range(len(remote_path_names)):
            path = '/' + '/'.join(remote_path_names[:i + 1])
            l.append(os.path.split(self.remote_files[path]['path'])[1])
        remote_path = '/' + '/'.join(l)
        remote_root_path = os.path.abspath(self.remote_root_path)
        local_path = remote_path.replace(remote_root_path + '/', '', 1)
        assert not local_path.startswith('/')
        return local_path

    # makes dirs if necessary, downloads, and adds to local state data
    def download(self, client, remote_path):
        remote_path = remote_path.lower()
        local_path = self.remote_path_to_local_path(client, remote_path)
        if not local_path:
            return 
        print '\tDownloading:', remote_path, 'into:', local_path
        # make the folder if it doesn't exist yet
        dirpath = os.path.split(local_path)[0]
        if dirpath:
            self.makedir_local(dirpath)
        with open(local_path,'w') as f:
            remote, meta = client.get_file_and_metadata(remote_path)
            shutil.copyfileobj(remote, f)
            remote.close()
        # add to local repository managed by local path
        self.local_files[remote_path] = {'modified': os.path.getmtime(local_path)}
        self.remote_files[remote_path] = meta
    
    def upload(self, client, local_path):
        remote_path = self.local_path_to_remote_path(client, local_path)
        print '\tUploading:', local_path, 'onto:', remote_path
        with open(local_path,'r') as f:
            meta = client.put_file(remote_path, f, True)
        
        self.local_files[remote_path] = {'modified': os.path.getmtime(local_path)}
        self.remote_files[remote_path] = meta
        
        # clean out the delta for the file upload
        self.execute_delta(client, ignore_path=remote_path)

    def delete(self, client, remote_path):
        print '\tFile deleted locally. Deleting on Dropbox:', remote_path
        try:
            client.file_delete(remote_path)
        except:
            # file was probably already deleted
            print '\tFile already removed from Dropbox'
            
        del self.local_files[remote_path]
        del self.remote_files[remote_path]

    # safely makes local dir
    def makedir_local(self, path, force=False):
        if not os.path.exists(path): # no need to make a dir that exists
            os.makedirs(path)
        elif os.path.isdir(path):
            pass
        elif os.path.isfile(path) and force: # if there is a file there ditch it
            os.remove(path)
            del self.files[path]
            
            os.makedir(path)
        else:
            raise IOError("regular file exists on the path")

    # recursively list files on dropbox
    def _listfiles(self, client, remote_path):
        meta = client.metadata(remote_path)
        filelist = []
        
        for item in meta['contents']:
            if item['is_dir']:
                filelist += self._listfiles(client, item['path'])
            else:
                filelist.append(item['path'])
        return filelist

    def download_all(self, client):
        filelist = self._listfiles(client, self.remote_root_path)
        for filepath in filelist:
            self.download(client, filepath)
    
    def check_state(self, client, local_path):
        remote_path = self.local_path_to_remote_path(client, local_path)
        # lets see if we've seen it before
        if remote_path not in self.local_files:
            # upload it!
            self.upload(client, local_path)
        elif os.path.getmtime(local_path) > self.local_files[remote_path]['modified']:
            # newer file than last sync
            self.upload(client, local_path)
        else:
            pass # looks like everything is good

    def ensure_remote_dir(self, client, remote_dir_path):
        try:
            client.file_create_folder(remote_dir_path)
        except ErrorResponse as e:
            if e.status != 403:
                raise


def loadstate():
    fyle = open(STATE_FILE,'r')
    state = pickle.load(fyle)
    fyle.close()
    
    return state


def savestate(state):
    fyle = open(STATE_FILE,'w')
    pickle.dump(state,fyle)
    fyle.close()
    

if __name__ == '__main__':
    import console

    # I moved 'dropboxlogin' into a sub folder so it doesn't clutter my main folder
    sys.path += [os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib')]
    import dropboxlogin # this code can be found here https://gist.github.com/4034526

    console.show_activity()

    print """
****************************************
*     Dropbox File Syncronization      *
****************************************"""

    client = dropboxlogin.get_client()

    print '\nLoading local state'
    # lets see if we can unpickle
    try:
        state = loadstate()
    except:
        print '\nCannot find state file. ***Making new local state***'
        # Aaaah, we have nothing, probably first run
        state = dropbox_state()

        print '\nDownloading everything from Dropbox'
        # no way to check what we have locally is newer, gratuitous dl
        state.ensure_remote_dir(client, REMOTE_ROOT_PATH)
        state.download_all(client)

    print '\nUpdating state from Dropbox'
    state.execute_delta(client)

    print '\nChecking for new or updated local files'
    # back to business, lets see if there is anything new or changed localy
    local_path_list = []
    for root, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            if filename != STATE_FILE:
                local_path_list.append(os.path.join(root, filename)[2:])

    for local_path in local_path_list:
        state.check_state(client, local_path)

    print '\nChecking for deleted local files'
    old_list = state.local_files.keys()
    for remote_path in old_list:
        local_path = state.remote_path_to_local_path(client, remote_path)
        if local_path not in local_path_list:
            state.delete(client, remote_path)
    
    print '\nSaving local state'
    savestate(state)
    
    print '\nSync complete'