Create a pyspark dataframe of filepaths by reading a directory in databricks

pyspark_df_from_fs_ls.md

#create a dataframe from filepaths

directory = '/path/on/dbfs'

file_paths = dbutils.fs.ls(directory)

#e.g
print(file_paths[0].path)
print(file_paths[0].name)

files_df = spark.createDataFrame(map(lambda path: (path.path,path.name), file_paths), ["path","name"])