CSV to Parquet using Spark

csv_to_parquet.scala

// sbt console

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.SparkSession


val sparkConf = new SparkConf().setAppName("test").setMaster("local[*]")
val sparkContext = new SparkContext(sparkConf)
val sparkSession = SparkSession.builder.config(sparkContext.getConf).getOrCreate()
val sqlContext = sparkSession.sqlContext
val df = sqlContext.read.format("com.databricks.spark.csv")
  .option("header", "true").option("inferSchema", "true")
  .option("charset", "UTF-8").load("path/to/file.csv")
df.write.option("compressionCodec", "gzip").parquet("path/to/file.parquet")