S3 Select Example

s3_select.py

import boto3
import pandas as pd

s3 = boto3.client('s3', 'eu-west-1')

def execute_query(query):
    response = s3.select_object_content(
        Bucket='my-bucket',
        Key='nyse/NYSE-2000-2001.tsv.gz',
        ExpressionType='SQL',
        Expression=query,
        InputSerialization={
            'CompressionType': 'GZIP',
            'CSV': {
                'FileHeaderInfo': 'IGNORE',
                'RecordDelimiter': '\n',
                'FieldDelimiter': '\t',
            }
        },
        OutputSerialization={
            'CSV': {
                #'RecordDelimiter': '\n',
                #'FieldDelimiter': ',',
            }
        }
    )

    for event in response['Payload']:
        if 'Records' in event:
            records = event['Records']['Payload'].decode('utf-8')
        elif 'Stats' in event:
            statsDetails = event['Stats']['Details']
            print("Stats details bytesScanned: ")
            print(statsDetails['BytesScanned'])
            print("Stats details bytesProcessed: ")
            print(statsDetails['BytesProcessed'])

    rows = [record.split(",") for record in records.split("\n")[:-1]]    
    return pd.DataFrame(rows)