Spark Session Create

spark_session_create.py

def create_spark_session():
    '''
    This function creates a spark session or finds an existing one and returns it
    Parameters - none
    Returns - spark session object
    '''
    
    spark = SparkSession \
                    .builder \
                    .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.0") \
                    .config("spark.hadoop.fs.s3a.impl","org.apache.hadoop.fs.s3a.S3AFileSystem") \
                    .getOrCreate()
    return spark

spark = create_spark_session()