Split data from one column to multiple columns using pyspark and remove first row which contains the headers

split_data.py

from pyspark.sql import SparkSession

spark = SparkSession.builder.master("local").appName('testapp').getOrCreate()

df = spark.createDataFrame([("col1:col2:col3",),
                            ("1:a:2001",),
                            ("2:b:2002",),
                            ("3:c:2003",)],
                           ["value"])
df.show()

df.createOrReplaceTempView("dftable")

df_split = spark.sql("select split(value,':') as column1 from dftable")
header = df_split.first()['column1']

df_split.show()

df_split = df_split.rdd.flatMap(lambda x: x).toDF(schema=header)

df_split = df_split.filter("col1 not like '%col1%'")

df_split.show()