Skip to content

Instantly share code, notes, and snippets.

@korakot
Last active November 12, 2020 01:04
Show Gist options
  • Save korakot/d56c925ff3eccb86ea5a16726a70b224 to your computer and use it in GitHub Desktop.
Save korakot/d56c925ff3eccb86ea5a16726a70b224 to your computer and use it in GitHub Desktop.
Google drive upload, read, update with Google Colab
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
# Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
# Create & upload a new file.
f = drive.CreateFile({'title': 'Sample file.txt'})
f.SetContentString('Sample upload file content') # or SetContentFile('im.png')
f.Upload() # see f.get('id')
# Read a file
fid = drive.ListFile({'q':"title='Sample file.txt'"}).GetList()[0]['id']
f = drive.CreateFile({'id': fid})
f.FetchMetadata(fetch_all=True) # some file types require this
text = f.GetContentString() # or f.GetContentFile('im.png') to save a local file
# Update a file
fid = drive.ListFile({'q':"title='Sample file.txt'"}).GetList()[0]['id']
f = drive.CreateFile({'id': fid})
f.SetContentString('Some newer content')
f.Upload()
# case when already known file_id, filename
drive.CreateFile({'id': file_id}).GetContentFile(filename)
# for csv, xlsx you need mimetype as a second argument
# 'text/csv' or 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
# google news vector example
file_id = '0B7XkCwpI5KDYNlNUTTlSS21pQmM'
downloaded = drive.CreateFile({'id':file_id})
downloaded.FetchMetadata(fetch_all=True)
downloaded.GetContentFile(downloaded.metadata['title'])
# combine upload new & update
res = drive.ListFile({'q':"title='Sample file.txt'"}).GetList()
if res:
fid = res[0]['id']
f = drive.CreateFile({'id': fid})
else:
f = drive.CreateFile({'title': 'Sample file.txt'})
f.SetContentString('Sample upload file content')
f.Upload()
# from https://stackoverflow.com/a/49433955/6729010
from google.colab import auth
from googleapiclient.http import MediaFileUpload
from googleapiclient.discovery import build
auth.authenticate_user() # click and fill
drive_service = build('drive', 'v3')
def save_file_to_drive(name, path):
file_metadata = {
'name': name,
'mimeType': 'application/octet-stream'
}
media = MediaFileUpload(path,
mimetype='application/octet-stream',
resumable=True)
created = drive_service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
print('File ID: {}'.format(created.get('id')))
return created
# then use the function
save_file_to_drive(destination_name, path_to_file)
# load credential file
import io, os
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
# get file_id
gdrive = build('drive', 'v3').files()
res = gdrive.list(q="name='service-account-colab.json'").execute()
file_id = res['files'][0]['id']
# download and save
req = gdrive.get_media(fileId=file_id)
fh = io.FileIO('service-account.json', 'wb')
downloader = MediaIoBaseDownload(fh, req)
while True:
status, done = downloader.next_chunk()
if done: break
# set credential
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/content/service-account.json'
!gdown --id $fid
@korakot
Copy link
Author

korakot commented Sep 2, 2018

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment