idiomer

'''
The path is a directory by default
'''

def hdfs_list(path, subtract_one=True):
    fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(spark._jsc.hadoopConfiguration())
    list_status = fs.listStatus(spark._jvm.org.apache.hadoop.fs.Path(path))
    # file.getPath().getName(), file.getBlockSize(), file.getLen()
    files_size = [file.getLen() for file in list_status]
    totol_size_in_MB = sum(files_size) / 1024.0 / 1024.0