Gridding radar data

grid_radar.py

import pyart
import cftime
import multiprocessing
import os
import sys

def grid_radar(infile):
    """
    Given the name of a radar file, create a file
    ``$PBS_JOBFS/gridded.{time}.nc`` containing a gridded representation of
    that data

    The output files may be concatenated using nco:
        
        ncrcat $PBS_JOBFS/gridded.*.nc gridded.nc
    """

    radar_in = pyart.io.read(infile)

    gatefilter = pyart.filters.GateFilter(radar_in)
    gatefilter.exclude_transition()
    gatefilter.exclude_masked('reflectivity')

    gridded_out = pyart.map.grid_from_radars(
            radar_in, gatefilters=gatefilter,
            grid_shape = (9,117,117),
            grid_limits=((0, 4000), (-150000.0, 150000.0), (-150000.0, 150000.0)),
            roi_func='constant', constant_roi=2500)

    # Grab the time from the data so we can add it to the output file name
    arttime = gridded_out.time
    time = cftime.num2date(arttime['data'][0], arttime['units'], calendar=arttime['calendar'])
    gridded_out.write(os.path.join(os.environ.get('PBS_JOBFS','.'), f'gridded.{time.strftime("%Y%m%dT%H%M%S")}.nc'))
    return time


def main():
    # Grid the input files in parallel using multiprocessing
    # PyART will run out of memory if we open too many files, so we only 
    # allow a single process to open 10 files, then we make a new one
    with multiprocessing.Pool(int(os.environ.get('PBS_NCPUS',4)), maxtasksperchild=10) as p:
        for r in p.imap_unordered(grid_radar, sys.argv[1:]):
            print(r)

if __name__ == '__main__':
    main()

grid_radar.sh

#!/bin/bash
#PBS -l ncpus=16
#PBS -l mem=32gb
#PBS -l walltime=0:20:00
#PBS -l wd
#PBS -l jobfs=2gb
#PBS -q express

# A PBS script to grid multiple radar files

INPUTS=/g/data/w97/sl1992/scott/cfrad_*_jepp_cdr_corrected_atten.nc
OUTPUT=gridded.nc

# Grid each input file - the outputs will be put in $PBS_JOBFS
module load conda/analysis3
python grid_radar.py $INPUTS > log

# Join the individual timesteps together with CDO
module load nco
ncrcat ${PBS_JOBFS:-.}/gridded.*.nc --output "$OUTPUT" --overwrite