delete_write_idempotency.py

import pandas as pd
import os
import shutil


def extract(path: str = "s3://my_bucket_name/file0.parquet") -> pd.DataFrame:
  df = pd.read_parquet(path)
  return df


# remove error rows
def transform(df: pd.DataFrame) -> pd.DataFrame:
  df_clean = df[df['customerId'] != 'A']
  return df_clean


def load_dwh(df: pd.DataFrame, output_location: str) -> None:
  if os.path.exists(output_location) == True:
    # removes the entire folder
    shutil.rmtree(output_location)
    os.makedir(output_location)
    df.to_csv(filename.csv)
  else:
    df.to_csv(filename.csv)
    
  return None