createChangedDataFrame.py

%%pyspark

schema = StructType([
    StructField("src_Id", IntegerType(), True),
    StructField("src_Firstname", StringType(), True),
    StructField("src_Lastname", StringType(), True),
    StructField("src_CreatedAt", DateType(), True),
    StructField("src_Department", StringType(), True),
    StructField("src_Salary", StringType(), True)
    ]
)

dataChanged = [
    Row(src_Id=1, src_Firstname='Jules', src_Lastname='Vega', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='Delivery', src_Salary='100000'),
    Row(src_Id=2, src_Firstname='Jummie', src_Lastname='Wallice', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='HR', src_Salary='100000'),
    Row(src_Id=3, src_Firstname='Hendrik', src_Lastname='Wolff', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='HR', src_Salary='100000'),
    Row(src_Id=4, src_Firstname='James', src_Lastname='Brauwn', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='HR', src_Salary='100000'),
    Row(src_Id=6, src_Firstname='Mia', src_Lastname='Black', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='HR', src_Salary='100000'),
    Row(src_Id=7, src_Firstname='Mike', src_Lastname='Hope', src_CreatedAt=datetime.strptime('2000-01-01', '%Y-%m-%d'), src_Department='Delivery', src_Salary='120000'),
]

dfDataChangedPath = "/TEST/Changed"
dfDataChanged = spark.createDataFrame(dataChanged, schema)
dfDataChanged.write.format("delta").mode("overwrite").save(dfDataChangedPath)

dfDataChanged.show()
dfDataChanged.printSchema()