PySpark: Set Custom Schema

custom_schema.py

# Reference: https://stackoverflow.com/questions/57901493/pyspark-defining-custom-schema-for-a-dataframe

table_schema = StructType([StructField('ID', StringType(), True),
                     StructField('Name', StringType(), True),
                     StructField('Tax_Percentage(%)', IntegerType(), False),
                     StructField('Effective_From', TimestampType(), False),
                     StructField('Effective_Upto', TimestampType(), True)])

df = spark.read.format(file_type) \
  .option("header", "true") \
  .option("sep", ",") \
  .schema(table_schema) \
  .load(file_location)