pyspark_rdd_read.py

# Filesystem schemes and URIs
|=====================================|
| Filesystem     | URI Structure      |
|----------------|--------------------|
| Local Fs       | file:///path       |
| HDFS           | hdfs://hdfs_path   |
| S3             | s3://bucket/object |
|=====================================|

# Loading a file into an RDD
rdd = sc.textFile("file:///filename")

# Loading a directory into rdd
rdd = sc.textFile("file:///dir/")

# Loading a directory with miltiple files
# in a tuple form ('filename', 'contents')
rdd = sc.wholeTextFile("file:///dir/")

# Loading all the CSVs from a directory
rdd = sc.textFile("file:///dir/*.csv")