pandas-on-spark-read-df.py

import pyspark.pandas as ps 

# data path in HDFS
loans_filename = '/FileStore/tables/loans.csv'

loans_df = ps.read_csv(
  loans_filename,
  header=None, 
  names=['loan_amount', 'address', 'created_at', 'funded_at'],        
  infer_datetime_format=True,
  parse_dates=['created_at', 'funded_at']
)