upload.py

gistfile1.py

import sys
import os
import shutil
import time
import dropbox
from dropbox.files import WriteMode, CommitInfo, UploadSessionCursor
from dropbox.exceptions import ApiError, AuthError

LOCALFILE = "toUpload/"
LOCALDIR = "toUpload"
COMPLETED_FOLDER = "completed"
ACCESS_TOKEN = ""
dbx = dropbox.Dropbox(ACCESS_TOKEN)


def ProcessFolder(**kwargs):
    # CHECKDIR if exists, if not create them
    # checkCurrentDir()
    files = []
    for root, dir, file in os.walk(LOCALDIR):
        for d in dir:
            createDirInCompletedFolder(os.path.relpath(os.path.join(root, d), LOCALDIR))

        for file_item in file:

            local_path = os.path.join(root, file_item)
            relative_path = os.path.relpath(local_path, LOCALDIR)
            dropbox_path = os.path.join(
                '\Cloud', "t", relative_path).replace("\\", "/")
            completed_path = os.path.join(COMPLETED_FOLDER, relative_path)
            filesize = os.stat(local_path).st_size

            # https://stackoverflow.com/a/47607204
            # is the file size above 100mb
            # print(100 << 20) gives you things in mb to bytes
            # break
            if filesize < 104857600:
                uploadIt(local_path, dropbox_path)
                time.sleep(20)
            else:
                print('File above 150mb so we need session upload')
                uploadLarge(local_path, dropbox_path)
                time.sleep(20)
            moveFile(relative_path)


def checkCurrentDir():
    if not os.chdir(LOCALDIR):
        os.mkdir(LOCALDIR)
    if not os.chdir(COMPLETED_FOLDER):
        os.mkdir(COMPLETED_FOLDER)


def createDirInCompletedFolder(relativeDIR):
    try:
        os.mkdir(os.path.join(COMPLETED_FOLDER, relativeDIR))
    except FileExistsError as error:
        print(error)


def moveFile(relative_path):
    fromDir = os.path.join(LOCALDIR, relative_path)
    toDir = os.path.join(COMPLETED_FOLDER, relative_path)
    print("Moving file from %s to %s" % (fromDir, toDir))
    shutil.move(fromDir, toDir)
    time.sleep(1)


def uploadLarge(file, destination):
    # https://stackoverflow.com/a/37399658
    # Set the destination globally for current file
    # setCommit(destination)

    # https://stackoverflow.com/a/30775393
    fSize = os.stat(file).st_size
    with open(file, 'rb') as f:
        for r in readTheLines(f, fSize, destination):
            # print(sys.getsizeof(r)/8388608)
            pass


def readTheLines(f, TotalSizeInBytes, destination):
    commit = dropbox.files.CommitInfo(path=destination)
    chunksize = 25*1024*1024
    TotalChunks = TotalSizeInBytes / chunksize  # returns number of chunks to process
    count = 0
    bytesProcessed = 0
    sessionID = None
    readData = None
    while bytesProcessed <= TotalSizeInBytes:
        CURSOR = UploadSessionCursor(session_id=sessionID, offset=f.tell())
        chunk = f.read(chunksize)  # return 8mb of data all the time

        count += 1

        if (TotalSizeInBytes - f.tell()) <= chunksize and readData is not None:
            # http://bit.ly/2G7rC2j
            dbx.files_upload_session_finish(chunk, CURSOR, commit)
            print("This is the last chunk")
            print('Finished a total of %d chunks' % count)
            break
            # add last
        # if this is the end of the data and there is no more return it
        if not chunk:
            print('Finished a total of %d chunks' % count)
            yield True
            break

        if readData is not None:
            # Data in between
            print('Processing intermediary chunk')
            readData = True
            # http://bit.ly/2XIR5cl
            dbx.files_upload_session_append_v2(chunk, CURSOR, False)
        else:
            # first chunk of data
            print('Processing First chunks')
            sessionID = dbx.files_upload_session_start(
                chunk).session_id
            readData = True
        bytesProcessed += chunksize
        print("A total of %d out of %d processed %d percent" %
              (bytesProcessed, TotalSizeInBytes, (bytesProcessed / TotalSizeInBytes)))
        time.sleep(20)


def uploadIt(file_location, destination):
    with open(file_location, 'rb') as f:
        # We use WriteMode=overwrite to make sure that the settings in the file
        # are changed on upload
        print("Uploading " + file_location +
              " to Dropbox as " + destination + "...")
        try:
            dbx.files_upload(
                f.read(), destination, mode=WriteMode('add'), autorename=True)
        except ApiError as err:
            # This checks for the specific error where a user doesn't have
            # enough Dropbox space quota to upload this file
            if (err.error.is_path() and
                    err.error.get_path().reason.is_insufficient_space()):
                sys.exit("ERROR: Cannot back up; insufficient space.")
            elif err.user_message_text:
                print(err.user_message_text)
                sys.exit()
            else:
                print(err)
                sys.exit()


ProcessFolder()