arrow_write_files.py

# Write to csv
df.to_csv("penguin-dataset.csv")

# Write to parquet
df.to_parquet("penguin-dataset.parquet")

# Write to Arrow
# Convert from pandas to Arrow
table = pa.Table.from_pandas(df)
# Write out to file
with pa.OSFile('penguin-dataset.arrow', 'wb') as sink:
    with pa.RecordBatchFileWriter(sink, table.schema) as writer:
        writer.write_table(table)

# Convert from no-NaN pandas to Arrow
table_nonan = pa.Table.from_pandas(df_nonan)
# Write out to file
with pa.OSFile('penguin-dataset-nonan.arrow', 'wb') as sink:
    with pa.RecordBatchFileWriter(sink, table_nonan.schema) as writer:
        writer.write_table(table_nonan)