read_grib_spark.py

# Read the GRIB file into a DataFrame
grib_df = spark.read.format("binaryFile").option("pathGlobFilter", "*.grb").load("/path/to/grib/file.grb")

# Define a function to read the GRIB file using cfgrib
def read_grib_file(file_path):
    with cfgrib.open_file(file_path) as ds:
        # Read the GRIB data into a pandas DataFrame
        grib_data = ds.to_dataframe()
        return grib_data

# Read the GRIB file using the defined function
grib_data = grib_df.rdd.map(lambda row: read_grib_file(row.path)).collect()