Skip to content

Instantly share code, notes, and snippets.

@smoll
Created January 8, 2019 16:33
Show Gist options
  • Save smoll/fcb3aa35bffdc0f4c92d29f81de80189 to your computer and use it in GitHub Desktop.
Save smoll/fcb3aa35bffdc0f4c92d29f81de80189 to your computer and use it in GitHub Desktop.
Testing gsutil's transparent compression of a csv file (and reading it back from the python lib)
import io
import csv
import sys
from google.cloud import storage
#
# Test download of gsutil gzipped csv file
#
def read_spike_file(bucket_name):
client = storage.Client()
bucket = client.get_bucket(bucket_name)
blob_iterator = bucket.list_blobs(prefix='spike.csv', delimiter='/')
for blob in blob_iterator:
print(f'blob: {blob}')
csv_file = io.StringIO(blob.download_as_string().decode('utf-8'))
for row in csv.DictReader(csv_file):
print(row)
if __name__ == '__main__':
try:
bucket_name = sys.argv[1]
except IndexError:
print('Missing bucket name!')
sys.exit(1)
read_spike_file(bucket_name)
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
google-cloud-storage = "*"
[requires]
python_version = "3.6"
col1 col2 col3
1 3 foo
2 5.01
-1 baz
#
# Test upload of gsutil gzipped csv file
#
: "${1?Missing bucket name!}"
gsutil rm "gs://$1/spike.csv"
gsutil cp -Z spike.csv "gs://$1/"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment