[Unzipping using Python & Pyspark] #Python #Spark #Pyspark

runner.py

# Using Python
import os, zipfile

z = zipfile.ZipFile('/databricks/driver/D-Dfiles.zip')
for f in z.namelist():
    if f.endswith('/'):
        os.makedirs(f)
        
# Reading zipped folder data in Pyspark

import zipfile
import io

def zip_extract(x):
    in_memory_data = io.BytesIO(x[1])
    file_obj = zipfile.ZipFile(in_memory_data, "r")
    files = [i for i in file_obj.namelist()]
    return dict(zip(files, [file_obj.open(file).read() for file in files]))


zips = sc.binaryFiles("dbfs:/mnt/vedant-demo/ONG/data/las_raw/D-Dfiles.zip")
files_data = zips.map(zip_extract)

Thank you for sharing. Do you know how to see the file in unzipped folder after your code? I tried files_data.collect() and saveAsTextFile, but both show errors.