Skip to content

Instantly share code, notes, and snippets.

@DrDanL
Created September 1, 2023 15:05
Show Gist options
  • Save DrDanL/9e7e73902a034613abe03bfb131f0d01 to your computer and use it in GitHub Desktop.
Save DrDanL/9e7e73902a034613abe03bfb131f0d01 to your computer and use it in GitHub Desktop.
Query and download data from Google Storage buckets using Python and pyrebase
import pyrebase
import os
# define firebase config for the project being queried
firebaseConfig = {
"apiKey": "",
"authDomain": "",
"projectId": "",
"storageBucket": "",
"messagingSenderId": "",
"appId": "",
"measurementId": "",
"serviceAccount": "",
"databaseURL":""
};
# create app connection using pyrebase based on firebase config
firebase_storage = pyrebase.initialize_app(firebaseConfig)
# create a storage bucket container
storage = firebase_storage.storage()
# obtain a list of files hosted on the storage bucket
all_files = storage.list_files()
# we create this as an index of files to ensure we keep a track on the number downloaded
cnt = 0
# define base storage location
base_url = '<BASE URL HERE>'
# loop through each file stored in all_files and download it
for file in all_files:
# let's check to see if the file already exists in its current location (it could be a file or folder)
if not os.path.exists(os.path.dirname(base_url + '/' + file.name)):
try:
# let make the directory if it doesn't already exist
os.makedirs(os.path.dirname(base_url + '/' + file.name))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
# the file exists pass and present a print
if(os.path.isfile(base_url + '/' + file.name)):
print('File present')
pass
else:
# file does not currently exist - download and store it
print(file.name + ' ' + str(cnt))
file.download_to_filename(base_url + '/' + file.name)
cnt += 1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment