NotebookCreateNewData.py

%%pyspark

from datetime import date
current_date = datetime.today().date()

try:
    # Read original data - this is your scd type 2 table holding all data
    dataOriginal = spark.read.load(dfDataOriginalPath + "/" + cw_database + "/" + cw_table, format='delta')
except:
    # Use first load when no data exists yet
    newOriginalData = dataChanged.withColumn('current', lit(True)).withColumn('effectiveDate', lit(current_date)).withColumn('endDate', lit(date(9999, 12, 31)))
    newOriginalData.write.format("delta").mode("overwrite").save(dfDataOriginalPath + "/" + cw_database + "/" + cw_table)
    newOriginalData.show()
    newOriginalData.printSchema()
    exit("Done loading data! Newly loaded data will be used to generate original data.")