stream_gcs_file.py

#Copyright 2022 Google LLC.
#SPDX-License-Identifier: Apache-2.0

import os
import psutil
from google.cloud.storage import Client

def get_mem_mb():
    return psutil.Process().memory_info().rss / (1024 * 1024)

class ChunkParser(object):
    def __init__(self, fileobj):
        self._fileobj = fileobj
        self.chunk_count = 0
        self.line_breaks_count = 0

    def write(self, chunk):
        #self._fileobj.write(chunk)
        line_breaks = chunk.count(b'\n')
        self.chunk_count += 1
        self.line_breaks_count += line_breaks

print(f'Allocated memory on start {get_mem_mb()}')

client = Client()
bucket = client.get_bucket('your-bucket')
blob = bucket.blob('inputs/data/compressed.csv')

with open('virtua_file', 'wb', os.O_NONBLOCK) as blob_file:
    parser = ChunkParser(blob_file)
    blob.download_to_file(parser)

    print(f'Total chunks {parser.chunk_count}')
    print(f'Total line breaks {parser.line_breaks_count}')
    print(f'Allocated memory on end {get_mem_mb()}')