s3fs.py

import s3fs

def stream_data_from_s3(bucket_name, file_path, chunk_size=1024):
    fs = s3fs.S3FileSystem()
    with fs.open(f'{bucket_name}/{file_path}', 'rb') as file:
        while chunk := file.read(chunk_size):
            yield chunk

# Example usage
bucket_name = 'my-bucket'
file_path = 'large_file.csv'
for chunk in stream_data_from_s3(bucket_name, file_path):
    # Process each chunk
    print(chunk)