Skip to content

Instantly share code, notes, and snippets.

@xflr6
Last active July 27, 2024 18:38
Show Gist options
  • Save xflr6/d106aa5b561fbac4ce1a9969eba728bb to your computer and use it in GitHub Desktop.
Save xflr6/d106aa5b561fbac4ce1a9969eba728bb to your computer and use it in GitHub Desktop.
Recursively traverse the directory tree of a Google Drive folder as variation of os.walk()
"""os.walk() variation with Google Drive API."""
import os
from apiclient.discovery import build # pip install google-api-python-client
FOLDER = 'application/vnd.google-apps.folder'
def get_credentials(scopes, *,
secrets='~/client_secrets.json',
storage='~/storage.json'):
from oauth2client import file, client, tools
store = file.Storage(os.path.expanduser(storage))
creds = store.get()
if creds is None or creds.invalid:
flow = client.flow_from_clientsecrets(os.path.expanduser(secrets), scopes)
flags = tools.argparser.parse_args([])
creds = tools.run_flow(flow, store, flags)
return creds
creds = get_credentials(scopes=['https://www.googleapis.com/auth/drive.metadata.readonly'])
service = build('drive', version='v3', credentials=creds)
def iterfiles(name=None, *, is_folder=None, parent=None,
order_by='folder,name,createdTime'):
q = []
if name is not None:
q.append("name = '{}'".format(name.replace("'", "\\'")))
if is_folder is not None:
q.append("mimeType {} '{}'".format('=' if is_folder else '!=', FOLDER))
if parent is not None:
q.append("'{}' in parents".format(parent.replace("'", "\\'")))
params = {'pageToken': None, 'orderBy': order_by}
if q:
params['q'] = ' and '.join(q)
while True:
response = service.files().list(**params).execute()
for f in response['files']:
yield f
try:
params['pageToken'] = response['nextPageToken']
except KeyError:
return
def walk(top='root', *, by_name: bool = False):
if by_name:
top, = iterfiles(name=top, is_folder=True)
else:
top = service.files().get(fileId=top).execute()
if top['mimeType'] != FOLDER:
raise ValueError(f'not a folder: {top!r}')
stack = [((top['name'],), top)]
while stack:
path, top = stack.pop()
dirs, files = is_file = [], []
for f in iterfiles(parent=top['id']):
is_file[f['mimeType'] != FOLDER].append(f)
yield path, top, dirs, files
if dirs:
stack.extend((path + (d['name'],), d) for d in reversed(dirs))
for kwargs in [{'top': 'spam', 'by_name': True}, {}]:
for path, root, dirs, files in walk(**kwargs):
print('/'.join(path), f'{len(dirs):d} {len(files):d}', sep='\t')
@john-delivuk
Copy link

One thing to note, if you have more then 1 folder with the same name this wont work. To accommodate I made a quick update.

def walk(top):
    if top:
        top = service.files().get(fileId=top).execute()
    else:
        top = service.files().get(fileId = "root").execute()
    stack = [((top['name'],), top)]
    while stack:
        path, top = stack.pop()
        dirs, files = is_file = [], []
        for f in iterfiles(parent=top['id']):
            is_file[f['mimeType'] != FOLDER].append(f)
        yield path, top, dirs, files
        if dirs:
            stack.extend((path + (d['name'],), d) for d in dirs)

To get the folder Id you can open google drive, then copy the value from the url.

@xflr6
Copy link
Author

xflr6 commented Dec 18, 2019

Thanks a lot @dbfanmanga and @john-delivuk for pointing this out (sorry for the late response).

Updated the signature of walk() to accept a folder id instead of a name per default (use by_name for the previous behaviour).
Using 'root' as default for walking complete drive contents.

@miwojc
Copy link

miwojc commented Feb 12, 2020

nice one. thanks!

as i needed to download files from share drive i added some args

top = service.files().get(fileId=top, supportsAllDrives=True).execute()

params = {
"pageToken": None,
"orderBy": order_by,
"supportsAllDrives": True,
"includeItemsFromAllDrives": True,
}

@sirusdas
Copy link

Thank You

@RaphaelPavan
Copy link

Hello there!

I am a new user to GDrive Python tools. Could you please give us an example of how ~/client_secrets.json should be?

An use case would be nice for us dummies to be able to set up this code, :)

I tried downloading a json on google's service account screen but it gave me the following error:

"InvalidClientSecretsError: Invalid file format. See https://developers.google.com/api-client-library/python/guide/aaa_client_secrets Expected a JSON object with a single property for a "web" or "installed" application"

@xflr6
Copy link
Author

xflr6 commented Jan 31, 2024

Hey, :) maybe this helps: https://medium.com/@ashokyogi5/a-beginners-guide-to-google-oauth-and-google-apis-450f36389184 (there is an example for a clients_secrets.json).

@MalikRumi
Copy link

https://github.com/googleapis/google-auth-library-python/blob/main/docs/index.rst

oauth2client was recently deprecated in favor of this library. For more details on the deprecation, see :doc:oauth2client-deprecation.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment