GCS example with pipes

gcs_example_with_pipes.py

import os
import threading

from google.cloud.storage import Client

PAYLOAD = b'DEADBEEF' * 100000


def reader(in_file):
    count = 0

    print("reader: start")
    while True:
        chunk = in_file.read(10000)
        print("reader: read one chunk")
        if not chunk:
            break
        count += len(chunk)

    print("reader: read {} bytes".format(count))


def main():
    client = Client()

    bucket = client.bucket('test-gcp-7218')
    if not bucket.exists():
        bucket.create()

    blob = bucket.blob('test.me')
    if not blob.exists():
        blob.upload_from_string(PAYLOAD)

    in_pipe, out_pipe = os.pipe()
    in_file = os.fdopen(in_pipe, mode='rb')
    out_file = os.fdopen(out_pipe, mode='wb')

    worker = threading.Thread(target=reader, args=(in_file,))
    worker.start()

    blob.download_to_file(out_file)

    out_file.close()
    worker.join()


if __name__ == '__main__':
    main()