benchmark_downloads_gcs.py

gistfile1.txt

from gcloud import storage
import bson
import StringIO
import csv
import gzip
import time
import numpy
from multiprocessing import Process


sizes_bytes = [10**x for x in range(0, 10)] #10

gcs_client = storage.Client(project='aclima-lab')
bucket = gcs_client.bucket('marco-performance-tests')

def download(gcs_file):
    blob = bucket.get_blob(gcs_file)
    download = blob.download_as_string()

for file_size in sizes_bytes:
    data = ['0' for x in xrange(0, file_size)]

    gcs_file = 'data_test_1/{}-test'.format(file_size)
    bucket.blob(gcs_file).upload_from_string(''.join(data))

    times = []
    procs = []
    for i in range(0, 10):
        start = time.time()
        procs.append(Process(target=download, args=(gcs_file)))
        procs[-1].start()
        stop = time.time()
        times.append(stop - start)
    for p in procs:
        p.join()
    print file_size, numpy.mean(times)