Download and upload files with GoogleDrive

download-and-upload-files-with-googledrive.ipynb

download-and-upload-files-with-googledrive.py

import os
from urllib import parse
import glob

# PyDrive
!pip install -U -q PyDrive
!pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive


# Authenticate and create the PyDrive client (need file 'client_secrets.json' to auth)
gauth = GoogleAuth()
gauth.CommandLineAuth() 
drive = GoogleDrive(gauth)


MIME_TYPES = {
    'application/vnd.google-apps.document':
    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    
    'application/vnd.google-apps.spreadsheet':
    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    
    'application/vnd.google-apps.presentation':
    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
    
    'application/vnd.google-apps.drawing':
    'application/pdf'
}


def downloadFromDriveFolder(gd_folder, local_parent_dir='.', verbose=True):
    local_folder_path = os.path.join(local_parent_dir, gd_folder['title'])

    if not os.path.exists(local_folder_path):
        os.makedirs(local_folder_path)
    
    gd_file_list = drive.ListFile({'q': "'{}' in parents and trashed=false".\
                                   format(gd_folder['id'])}).GetList()

    for gd_file in gd_file_list:
        if gd_file['mimeType']=='application/vnd.google-apps.folder':
            if verbose:
                print(os.path.join(local_folder_path, gd_file['title']))
                
            downloadFromDriveFolder(gd_file, local_folder_path, verbose)
        else:
            if verbose:
                print(os.path.join(local_folder_path, gd_file['title']))
                
            if gd_file['mimeType'] in MIME_TYPES:
                gd_file.GetContentFile(os.path.join(local_folder_path, gd_file['title']),
                                       mimetype=MIME_TYPES[gd_file['mimeType']])
            else:
                gd_file.GetContentFile(os.path.join(local_folder_path, gd_file['title']))

                
def downloadByDriveURL(url, local_parent_dir='.', verbose=True):
    '''
    Method to download a file or folder from the GoogleDrive.

    :param url: file or folder url in GoogleDrive
    :param local_parent_dir: local parent directory that will be created if it does not exist
    :param verbose: print downloading steps
    '''
    gd_file_id = parse.parse_qs(parse.urlparse(url).query)['id'][0]
    gd_file = drive.CreateFile({'id': gd_file_id})
    
    if gd_file['mimeType']=='application/vnd.google-apps.folder':
        if verbose:
            print('Downloading folder "{}"...'.format(gd_file['title']))
        
        downloadFromDriveFolder(gd_file, local_parent_dir, verbose)
    else:
        if not os.path.exists(local_parent_dir):
            os.makedirs(local_parent_dir)

        if verbose:
            print('Downloading file "{}"...'.format(gd_file['title']))
            print(os.path.join(local_parent_dir, gd_file['title']))
        
        if gd_file['mimeType'] in MIME_TYPES:
            gd_file.GetContentFile(os.path.join(local_parent_dir, gd_file['title']),
                                   mimetype=MIME_TYPES[gd_file['mimeType']])
        else:
            gd_file.GetContentFile(os.path.join(local_parent_dir, gd_file['title']))


def uploadToDriveFolder(local_file_or_dir, gd_parent_folder=None, verbose=True):
    if verbose:
        print(local_file_or_dir)
            
    local_basename = os.path.basename(local_file_or_dir)            
    if os.path.isfile(local_file_or_dir):
        if gd_parent_folder is None:
            gd_file = drive.CreateFile({'title': local_basename})
        else:
            gd_file = drive.CreateFile(
                {'title': local_basename,
                 'parents': [{'kind': 'drive#fileLink', 'id': gd_parent_folder['id']}]})
        
        gd_file.SetContentFile(local_file_or_dir)
        gd_file.Upload()
        
    elif os.path.isdir(local_file_or_dir):
        if gd_parent_folder is None:
            gd_folder = drive.CreateFile(
                {'title': local_basename,
                 'mimeType': 'application/vnd.google-apps.folder'})
        else:
            gd_folder = drive.CreateFile(
                {'title': local_basename,
                 'mimeType': 'application/vnd.google-apps.folder',
                 'parents': [{'kind': 'drive#fileLink', 'id': gd_parent_folder['id']}]})
        
        gd_folder.Upload()
        for local_path in glob.iglob(os.path.join(local_file_or_dir,'*')):
            uploadToDriveFolder(local_path, gd_folder)


def uploadToDrive(local_file_or_dir, gd_parent_folder_name=None,gd_parent_folder_url=None,
                  verbose=True):
    '''
    Method to upload a file or folder to the GoogleDrive.
    
    Set one of two parameters "gd_parent_folder_name"
    or "gd_parent_folder_url" to specify the parent folder.
    
    If the parameters "gd_parent_folder_name" and "gd_parent_folder_url" are not defined,
    the files are uploaded to the root folder of the user.

    :param local_file_or_dir: local file or directory to upload to GoogleDrive
    :param gd_parent_folder_name: name of parent folder that will be created in GoogleDrive
    :param gd_parent_folder_url: url of existing parent folder in GoogleDrive
    :param verbose: print uploading steps
    '''
    if not os.path.exists(local_file_or_dir):
        raise Exception('Local file or directory not found"!')
        
    if gd_parent_folder_url is not None and gd_parent_folder_name is not None:
        raise Exception('Set one of two parameters: "gd_parent_folder_url"'\
                        ' or "gd_parent_folder_name"!')
    
    if gd_parent_folder_url is not None:
        gd_parent_folder_id = parse.parse_qs(parse.urlparse(gd_parent_folder_url).query)['id'][0]
        gd_parent_folder = drive.CreateFile({'id': gd_parent_folder_id})
        if gd_parent_folder['mimeType']!='application/vnd.google-apps.folder':
            raise Exception('"{}" is not folder!'.format(gd_parent_folder['title']))
        if verbose:
            print('Uploading "{}" to "{}" by folder url...'.\
                  format(local_file_or_dir, gd_parent_folder['title']))
        uploadToDriveFolder(local_file_or_dir, gd_parent_folder, verbose)
        
    elif gd_parent_folder_name is not None:
        if verbose:
            print('Uploading "{}" to "{}" by folder name...'.\
                  format(local_file_or_dir, gd_parent_folder_name))
        gd_parent_folder = drive.CreateFile(
            {'title': gd_parent_folder_name,
             'mimeType': 'application/vnd.google-apps.folder'})
        gd_parent_folder.Upload()
        uploadToDriveFolder(local_file_or_dir, gd_parent_folder, verbose)
        
    else:
        if verbose:
            print('Uploading "{}" to root GoogleDrive...'.format(local_file_or_dir))
        uploadToDriveFolder(local_file_or_dir,verbose=verbose)

download-and-upload-files-with-googledrive_colab.py

import os
from urllib import parse
import glob

# PyDrive
!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials


# Authenticate and create the PyDrive client
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)


MIME_TYPES = {
    'application/vnd.google-apps.document':
    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
    
    'application/vnd.google-apps.spreadsheet':
    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    
    'application/vnd.google-apps.presentation':
    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
    
    'application/vnd.google-apps.drawing':
    'application/pdf'
}


def downloadFromDriveFolder(gd_folder, local_parent_dir='.', verbose=True):
    local_folder_path = os.path.join(local_parent_dir, gd_folder['title'])

    if not os.path.exists(local_folder_path):
        os.makedirs(local_folder_path)
    
    gd_file_list = drive.ListFile({'q': "'{}' in parents and trashed=false".\
                                   format(gd_folder['id'])}).GetList()

    for gd_file in gd_file_list:
        if gd_file['mimeType']=='application/vnd.google-apps.folder':
            if verbose:
                print(os.path.join(local_folder_path, gd_file['title']))
                
            downloadFromDriveFolder(gd_file, local_folder_path, verbose)
        else:
            if verbose:
                print(os.path.join(local_folder_path, gd_file['title']))
                
            if gd_file['mimeType'] in MIME_TYPES:
                gd_file.GetContentFile(os.path.join(local_folder_path, gd_file['title']),
                                       mimetype=MIME_TYPES[gd_file['mimeType']])
            else:
                gd_file.GetContentFile(os.path.join(local_folder_path, gd_file['title']))

                
def downloadByDriveURL(url, local_parent_dir='.', verbose=True):
    '''
    Method to download a file or folder from the GoogleDrive.

    :param url: file or folder url in GoogleDrive
    :param local_parent_dir: local parent directory that will be created if it does not exist
    :param verbose: print downloading steps
    '''
    gd_file_id = parse.parse_qs(parse.urlparse(url).query)['id'][0]
    gd_file = drive.CreateFile({'id': gd_file_id})
    
    if gd_file['mimeType']=='application/vnd.google-apps.folder':
        if verbose:
            print('Downloading folder "{}"...'.format(gd_file['title']))
        
        downloadFromDriveFolder(gd_file, local_parent_dir, verbose)
    else:
        if not os.path.exists(local_parent_dir):
            os.makedirs(local_parent_dir)

        if verbose:
            print('Downloading file "{}"...'.format(gd_file['title']))
            print(os.path.join(local_parent_dir, gd_file['title']))
        
        if gd_file['mimeType'] in MIME_TYPES:
            gd_file.GetContentFile(os.path.join(local_parent_dir, gd_file['title']),
                                   mimetype=MIME_TYPES[gd_file['mimeType']])
        else:
            gd_file.GetContentFile(os.path.join(local_parent_dir, gd_file['title']))


def uploadToDriveFolder(local_file_or_dir, gd_parent_folder=None, verbose=True):
    if verbose:
        print(local_file_or_dir)
            
    local_basename = os.path.basename(local_file_or_dir)            
    if os.path.isfile(local_file_or_dir):
        if gd_parent_folder is None:
            gd_file = drive.CreateFile({'title': local_basename})
        else:
            gd_file = drive.CreateFile(
                {'title': local_basename,
                 'parents': [{'kind': 'drive#fileLink', 'id': gd_parent_folder['id']}]})
        
        gd_file.SetContentFile(local_file_or_dir)
        gd_file.Upload()
        
    elif os.path.isdir(local_file_or_dir):
        if gd_parent_folder is None:
            gd_folder = drive.CreateFile(
                {'title': local_basename,
                 'mimeType': 'application/vnd.google-apps.folder'})
        else:
            gd_folder = drive.CreateFile(
                {'title': local_basename,
                 'mimeType': 'application/vnd.google-apps.folder',
                 'parents': [{'kind': 'drive#fileLink', 'id': gd_parent_folder['id']}]})
        
        gd_folder.Upload()
        for local_path in glob.iglob(os.path.join(local_file_or_dir,'*')):
            uploadToDriveFolder(local_path, gd_folder)


def uploadToDrive(local_file_or_dir, gd_parent_folder_name=None,gd_parent_folder_url=None,
                  verbose=True):
    '''
    Method to upload a file or folder to the GoogleDrive.
    
    Set one of two parameters "gd_parent_folder_name"
    or "gd_parent_folder_url" to specify the parent folder.
    
    If the parameters "gd_parent_folder_name" and "gd_parent_folder_url" are not defined,
    the files are uploaded to the root folder of the user.

    :param local_file_or_dir: local file or directory to upload to GoogleDrive
    :param gd_parent_folder_name: name of parent folder that will be created in GoogleDrive
    :param gd_parent_folder_url: url of existing parent folder in GoogleDrive
    :param verbose: print uploading steps
    '''
    if not os.path.exists(local_file_or_dir):
        raise Exception('Local file or directory not found"!')
        
    if gd_parent_folder_url is not None and gd_parent_folder_name is not None:
        raise Exception('Set one of two parameters: "gd_parent_folder_url"'\
                        ' or "gd_parent_folder_name"!')
    
    if gd_parent_folder_url is not None:
        gd_parent_folder_id = parse.parse_qs(parse.urlparse(gd_parent_folder_url).query)['id'][0]
        gd_parent_folder = drive.CreateFile({'id': gd_parent_folder_id})
        if gd_parent_folder['mimeType']!='application/vnd.google-apps.folder':
            raise Exception('"{}" is not folder!'.format(gd_parent_folder['title']))
        if verbose:
            print('Uploading "{}" to "{}" by folder url...'.\
                  format(local_file_or_dir, gd_parent_folder['title']))
        uploadToDriveFolder(local_file_or_dir, gd_parent_folder, verbose)
        
    elif gd_parent_folder_name is not None:
        if verbose:
            print('Uploading "{}" to "{}" by folder name...'.\
                  format(local_file_or_dir, gd_parent_folder_name))
        gd_parent_folder = drive.CreateFile(
            {'title': gd_parent_folder_name,
             'mimeType': 'application/vnd.google-apps.folder'})
        gd_parent_folder.Upload()
        uploadToDriveFolder(local_file_or_dir, gd_parent_folder, verbose)
        
    else:
        if verbose:
            print('Uploading "{}" to root GoogleDrive...'.format(local_file_or_dir))
        uploadToDriveFolder(local_file_or_dir,verbose=verbose)