100mb parallel upload to dropbox and drive

eks.py

#!/usr/bin/python -u

"""
Dropbox File Transfer Comparison - POC

This program compares the speed of uploading multiple files to dropbox using
both queued and parallel connections per file.
"""

import time
import threading
import dropbox
import sys


import httplib2
import pprint

from apiclient import errors
from apiclient.discovery import build
from apiclient.http import MediaFileUpload
from oauth2client.client import OAuth2WebServerFlow

app_key = 'fdftzwseyoqxzx1'
app_secret = 'mffquu98zosmk45'


class UploadThreadDropbox(threading.Thread):
    def __init__(self, client, file, n):
        threading.Thread.__init__(self)
        self.client = client
        self.file = file
        self.n = n
        self.retry = False

    def run(self):
        while True:
            f = open(self.file, 'rb')
            try:
                response = self.client.put_file('/tmp/p%s.jpg' % self.n, f)
                self.retry=False
                break
            except Exception as e:
                print 'EXCEPTION %s' % e
                self.retry=True

class UploadThreadDrive(threading.Thread):
    def __init__(self, cliente, file, n):
        threading.Thread.__init__(self)
        self.cliente = cliente
        self.file = file
        self.n = n
        self.retry = False

    def run(self):
        while True:
            try:
                file = insert_file(self.cliente, self.file)
                print 'END'
                self.retry=False
                break
            except Exception as e:
                print 'EXCEPTION %s' % e
                self.retry=True

def create_file(i, fileSize):
    with open("files/" + str(i), "wb") as out:
            out.truncate(fileSize)
    return "files/" + str(i)


def upload_in_parallel_dropbox(client, files):
    retry = True
    start_time = 0
    while retry:
        start_time = time.time()
        retry = False
        threads = []
        for (i, file) in enumerate(files):
            threads.append(UploadThreadDropbox(client, file, i))
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
            if thread.retry:
                retry = True
                print 'RETRY'

    time_taken = time.time() - start_time
    print 'dropbox in parallel time taken (seconds): %s' % round(time_taken, 5)

def upload_in_both(db, dr, files):
    retry = True
    start_time = 0
    while retry:
        start_time = time.time()
        retry = False
        threads = []
        threads.append(UploadThreadDrive(dr, files[0], 0))
        threads.append(UploadThreadDropbox(db, files[1], 1))
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
            if thread.retry:
                retry = True
                print 'RETRY'

    time_taken = time.time() - start_time
    print 'both in parallel time taken (seconds): %s' % round(time_taken, 5)

def upload_in_parallel_drive(cliente, files):
    retry = True
    start_time = 0
    while retry:
        start_time = time.time()
        retry = False
        threads = []
        for (i, file) in enumerate(files):
            threads.append(UploadThreadDrive(cliente, file, i))
        for thread in threads:
            thread.start()
            thread.join()
        for thread in threads:
            if thread.retry:
                retry = True
                print 'RETRY'

    time_taken = time.time() - start_time
    print 'drive in parallel time taken (seconds): %s' % round(time_taken, 5)

def insert_file(service, filename):
  """Insert new file.

  Args:
    service: Drive API service instance.
    title: Title of the file to insert, including the extension.
    description: Description of the file to insert.
    parent_id: Parent folder's ID.
    mime_type: MIME type of the file to insert.
    filename: Filename of the file to insert.
  Returns:
    Inserted file metadata if successful, None otherwise.
  """

  media_body = MediaFileUpload(filename, mimetype="application/octet-stream", resumable=False)
  body = {
    'title': filename,
    'description': "testfile",
    'mimeType': "application/octet-stream"
  }

  try:
    file = service.files().insert(
        body=body,
        media_body=media_body).execute()

    # Uncomment the following line to print the File ID
    # print 'File ID: %s' % file['id']

    return file
  except errors.HttpError, error:
    print 'An error occured: %s' % error.read()
    return None

if __name__ == "__main__":
# Copy your credentials from the console
    CLIENT_ID = '86758202287-0v80bb7019tbnh6ttqvmo7lhusn1hlc5.apps.googleusercontent.com'
    CLIENT_SECRET = '_IfUC3Y3C4xvj_NztTZa0IAF'

# Check https://developers.google.com/drive/scopes for all available scopes
    OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'

# Redirect URI for installed apps
    REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'

# Run through the OAuth flow and retrieve credentials
    flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE,
                               redirect_uri=REDIRECT_URI)
    authorize_url = flow.step1_get_authorize_url()
    print 'Go to the following link in your browser: ' + authorize_url
    code = raw_input('Enter verification code: ').strip()
    credentials = flow.step2_exchange(code)

# Create an httplib2.Http object and authorize it with our credentials
    http = httplib2.Http()
    http = credentials.authorize(http)
    http.disable_ssl_certificate_validation = True
    http.force_exception_to_status_code = True

    drive_service = build('drive', 'v2', http=http)




    flow = dropbox.client.DropboxOAuth2FlowNoRedirect(app_key, app_secret)

    authorize_url = flow.start()
    if len(sys.argv) == 2:
        code = sys.argv[1]
    else:
        print '1. Go to: ' + authorize_url
        print '2. Click "Allow" (you might have to log in first)'
        print '3. Copy the authorization code.'
        code = raw_input("Enter the authorization code here: ").strip()




    try:
        access_token, user_id = flow.finish(code)

        client = dropbox.client.DropboxClient(access_token)
        print 'linked account: ', client.account_info()['display_name']

        fileCounts = [2]
        sizes = [1024 * 1024 * 100]
        for size in sizes:
            for count in fileCounts:
                print 'Uploading ' + str(count) +  ' files of size ' + str(size)
                files = []
                for i in range(count):
                    files.append(create_file(i, size))
                upload_in_parallel_dropbox(client, files)
                time.sleep(1)
                upload_in_parallel_drive(drive_service, files)
                time.sleep(1)
                upload_in_both(client, drive_service, files)
                time.sleep(1)

    except dropbox.rest.ErrorResponse:
        print 'invalid authorization code.'

res.txt

Uploading 2 files of size 104857600
dropbox in parallel time taken (seconds): 46.07439
drive in parallel time taken (seconds): 23.1675
both in parallel time taken (seconds): 46.18402

Uploading 2 files of size 104857600
dropbox in parallel time taken (seconds): 49.53807
drive in parallel time taken (seconds): 24.41576
both in parallel time taken (seconds): 45.55829

Uploading 2 files of size 104857600
dropbox in parallel time taken (seconds): 48.13655
drive in parallel time taken (seconds): 24.30074
both in parallel time taken (seconds): 46.54713