Pyspark vs Polars Utils

_README.md

Polars vs Pyspark Utils

The following files are util functions for easier Polars and Pyspark conversion development.

from_csv.py

import boto3
import polars from pl

client = boto3.client('s3')

def from_csv(bucket_name, input_path):
    data = client.get_object(Bucket=bucket_name, Key=input_path)
    csv_bytes = data['Body'].read()
    return pl.read_csv(csv_bytes)

pyspark_to_polars.py

import polars from pl

# Assumes pandas and pyspark are installed
def to_polars(spark_df):
    pandas_df = spark_df.select("*").toPandas()
    data = pandas_df.to_dict('list')
    return pl.DataFrame(data)

to_csv.py

import boto3
import polars from pl

client = boto3.client('s3')

def to_csv(df, bucket_name, output_path):
    client.put_object(Body=df.write_csv(), Bucket=bucket_name, Key=output_path)