Code to read S-band data and retrieve KDP field using lp_phase_proc algorithm (Giangrande et al., 2012) using PyART gatefilter version

lp_phase_proc_kdp_gatefilter_version

import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from matplotlib import dates
import pyart
import glob
import pandas as pd
from scipy import ndimage, interpolate
import copy
import matplotlib.cm as cm
import skfuzzy as fuzz
%matplotlib inline

## Copy pasted function definitions from cmac.... Please jump tp line 266 (actual code starts there)

def cum_score_fuzzy_logic(radar, mbfs=None,
                          ret_scores=False,
                          hard_const=None,
                          verbose=False):
    if mbfs is None:
        second_trip = {'velocity_texture': [[0, 0, 1.8, 2], 1.0],
                       'cross_correlation_ratio': [[.5, .7, 1, 1], 0.0],
                       'normalized_coherent_power': [[0, 0, .5, .6], 3.0],
                       'height': [[0, 0, 5000, 8000], 1.0],
                       'sounding_temperature': [[-100, -100, 100, 100], 0.0],
                       'signal_to_noise_ratio': [[15, 20, 1000, 1000], 1.0]}

        rain = {'differential_phase_texture': [[0, 0, 80, 90], 1.0],
                'cross_correlation_ratio': [[0.94, 0.96, 1, 1], 1.0],
                'normalized_coherent_power': [[0.4, 0.5, 1, 1], 1.0],
                'height': [[0, 0, 5000, 6000], 0.0],
                'sounding_temperature': [[0, 3, 100, 100], 2.0],
                'signal_to_noise_ratio': [[8, 10, 1000, 1000], 1.0]}

        snow = {'differential_phase_texture': [[0, 0, 80, 90], 1.0],
                'cross_correlation_ratio': [[0.85, 0.9, 1, 1], 1.0],
                'normalized_coherent_power': [[0.4, 0.5, 1, 1], 1.0],
                'height': [[0, 0, 25000, 25000], 0.0],
                'sounding_temperature': [[-100, -100, 0, 1.], 2.0],
                'signal_to_noise_ratio': [[8, 10, 1000, 1000], 1.0]}

        no_scatter = {'differential_phase_texture': [[90, 90, 400, 400], 0.0],
                      'cross_correlation_ratio': [[0, 0, 0.1, 0.2], 0.0],
                      'normalized_coherent_power': [[0, 0, 0.1, 0.2], 0.0],
                      'height': [[0, 0, 25000, 25000], 0.0],
                      'sounding_temperature': [[-100, -100, 100, 100], 0.0],
                      'signal_to_noise_ratio': [[-100, -100, 8, 10], 6.0]}

        melting = {'differential_phase_texture': [[20, 30, 80, 90], 0.0],
                   'cross_correlation_ratio': [[0.6, 0.7, .94, .96], 4.],
                   'normalized_coherent_power': [[0.4, 0.5, 1, 1], 0],
                   'height': [[0, 0, 25000, 25000], 0.0],
                   'sounding_temperature': [[-1., 0, 3.5, 5], 2.],
                   'signal_to_noise_ratio': [[8, 10, 1000, 1000], 0.0]}

        mbfs = {'multi_trip': second_trip, 'rain': rain, 'snow': snow,
                'no_scatter': no_scatter, 'melting': melting}

    flds = radar.fields
    scores = {}
    for key in mbfs.keys():
        if verbose:
            print('##    Doing', key)
        this_score = np.zeros(
            flds[list(flds.keys())[0]]['data'].shape).flatten() * 0.0
        for MBF in mbfs[key].keys():
            this_score = fuzz.trapmf(
                flds[MBF]['data'].flatten(),
                mbfs[key][MBF][0]) * mbfs[key][MBF][1] + this_score

        this_score = this_score.reshape(
            flds[list(flds.keys())[0]]['data'].shape)
        scores.update({key: ndimage.filters.median_filter(
            this_score, size=[3, 4])})

    if hard_const is not None:
        # hard_const = [[class, field, (v1, v2)], ...]
        for this_const in hard_const:
            if verbose:
                print('##    Doing hard constraining', this_const[0])
            key = this_const[0]
            const = this_const[1]
            fld_data = radar.fields[const]['data']
            lower = this_const[2][0]
            upper = this_const[2][1]
            const_area = np.where(np.logical_and(fld_data >= lower,
                                                 fld_data <= upper))
            if verbose:
                print('##    ', str(const_area))
            scores[key][const_area] = 0.0
    stacked_scores = np.dstack([scores[key] for key in scores.keys()])
    #sum_of_scores = stacked_scores.sum(axis = 2)
    #print(sum_of_scores.shape)
    #norm_stacked_scores = stacked_scores
    max_score = stacked_scores.argmax(axis=2)

    gid = {}
    gid['data'] = max_score
    gid['units'] = ''
    gid['standard_name'] = 'gate_id'
    strgs = ''
    i = 0
    for key in scores.keys():
        strgs = strgs + str(i) + ':' + key + ','
        i = i + 1
    gid['long_name'] = 'Classification of dominant scatterer'
    gid['notes'] = strgs[0:-1]
    gid['valid_max'] = max_score.max()
    gid['valid_min'] = 0.0

    if ret_scores is False:
        rv = (gid, scores.keys())
    else:
        rv = (gid, scores.keys(), scores)
    return rv



def get_melt(radar, melt_cat=None):
    if melt_cat is None:
        cat_dict = {}
        for pair_str in radar.fields['gate_id']['notes'].split(','):
            cat_dict.update(
                {pair_str.split(':')[1]: int(pair_str.split(':')[0])})

        melt_cat = cat_dict['melting']

    melt_locations = np.where(radar.fields['gate_id']['data'] == melt_cat)
    kinda_cold = np.where(radar.fields['sounding_temperature']['data'] < 0)
    fzl_sounding = radar.gate_altitude['data'][kinda_cold].min()
    if len(melt_locations[0] > 1):
        fzl_pid = radar.gate_altitude['data'][melt_locations].min()
        fzl = (fzl_pid + fzl_sounding) / 2.0
    else:
        fzl = fzl_sounding

    print(fzl)
    if fzl > 5000:
        fzl = 3500.0
    if fzl < 1000:
        fzl = radar.gate_altitude['data'].min()
    return fzl



def _fix_rain_above_bb(gid_fld, rain_class, melt_class, snow_class):
    print(snow_class)
    new_gid = copy.deepcopy(gid_fld)
    for ray_num in range(new_gid['data'].shape[0]):
        if melt_class in new_gid['data'][ray_num, :]:
            max_loc = np.where(
                new_gid['data'][ray_num, :] == melt_class)[0].max()
            rain_above_locs = np.where(
                new_gid['data'][ray_num, max_loc:] == rain_class)[0] + max_loc
            new_gid['data'][ray_num, rain_above_locs] = snow_class
    return new_gid



def do_my_fuzz(radar, rhv_field, ncp_field,
               tex_start=2.0, tex_end=2.1,
               custom_mbfs=None, custom_hard_constraints=None,
               verbose=True):  # NEEDS DOCSTRING
    if verbose:
        print('##')
        print('## CMAC calculation using fuzzy logic:')

    second_trip = {'velocity_texture': [[tex_start, tex_end, 130., 130.], 4.0],
                   rhv_field: [[.5, .7, 1, 1], 0.0],
                   ncp_field: [[0, 0, .5, .6], 1.0],
                   'height': [[0, 0, 5000, 8000], 0.0],
                   'sounding_temperature': [[-100, -100, 100, 100], 0.0],
                   'signal_to_noise_ratio': [[5, 10, 1000, 1000], 1.0]}

    rain = {'velocity_texture': [[0, 0, tex_start, tex_end], 1.0],
            rhv_field: [[0.97, 0.98, 1, 1], 1.0],
            ncp_field: [[0.4, 0.5, 1, 1], 1.0],
            'height': [[0, 0, 5000, 6000], 0.0],
            'sounding_temperature': [[2., 5., 100, 100], 2.0],
            'signal_to_noise_ratio': [[8, 10, 1000, 1000], 1.0]}

    snow = {'velocity_texture': [[0, 0, tex_start, tex_end], 1.0],
            rhv_field: [[0.65, 0.9, 1, 1], 1.0],
            ncp_field: [[0.4, 0.5, 1, 1], 1.0],
            'height': [[0, 0, 25000, 25000], 0.0],
            'sounding_temperature': [[-100, -100, .5, 4.], 2.0],
            'signal_to_noise_ratio': [[8, 10, 1000, 1000], 1.0]}

    no_scatter = {'velocity_texture': [[tex_start, tex_end, 330., 330.], 2.0],
                  rhv_field: [[0, 0, 0.1, 0.2], 0.0],
                  ncp_field: [[0, 0, 0.1, 0.2], 0.0],
                  'height': [[0, 0, 25000, 25000], 0.0],
                  'sounding_temperature': [[-100, -100, 100, 100], 0.0],
                  'signal_to_noise_ratio': [[-100, -100, 5, 10], 4.0]}

    melting = {'velocity_texture': [[0, 0, tex_start, tex_end], 0.0],
               rhv_field: [[0.6, 0.65, .9, .96], 2.0],
               ncp_field: [[0.4, 0.5, 1, 1], 0],
               'height': [[0, 0, 25000, 25000], 0.0],
               'sounding_temperature': [[0, 0.1, 2, 4], 4.0],
               'signal_to_noise_ratio': [[8, 10, 1000, 1000], 0.0]}

    if custom_mbfs is None:
        mbfs = {'multi_trip': second_trip, 'rain': rain, 'snow': snow,
                'no_scatter': no_scatter, 'melting': melting}
    else:
        mbfs = custom_mbfs

    if custom_hard_constraints is None:
        hard_const = [['melting', 'sounding_temperature', (10, 100)],
                      ['multi_trip', 'height', (10000, 1000000)],
                      ['melting', 'sounding_temperature', (-10000, -2)],
                      ['rain', 'sounding_temperature', (-1000, -5)],
                      ['melting', 'velocity_texture', (3, 300)]]
    else:
        hard_const = custom_hard_constraints

    gid_fld, cats = cum_score_fuzzy_logic(radar, mbfs=mbfs, verbose=verbose,
                                          hard_const=hard_const)
    rain_val = list(cats).index('rain')
    snow_val = list(cats).index('snow')
    melt_val = list(cats).index('melting')
    return _fix_rain_above_bb(gid_fld, rain_val, melt_val, snow_val), cats

def gen_clutter_field_from_refl(radar, corrected_field, uncorrected_field, diff_dbz=-12.0, max_h=2000.0):
    """
    Generate a Py-ART field with clutter flagging using
    differences in reflectivity. A cludge for when you
    have a clutter corrected field but no field with the
    flags
    Parameters
    ----------
    radar : Radar
        A radar object to create the clutter field from
    corrected_field : String
        Field name for a field which has had gates with clutter in them reduced some how
    uncorrected_field : String
        Field name for raw reflectivity field
    diff_dbz : float
        Difference in dBZ below which a gate is considered clutter. Defaiults to -12dBZ
    max_h : float
        Height (in m) above which all gates are considered to be clutter free.
    Returns
    clutter_field : Dict
        A field dictionary whith an entry 'data' where clutter is flagged as '1' and clutter
        free is flagged as '0'
    """
    new_grid = radar.fields[corrected_field]['data'] - radar.fields[uncorrected_field]['data']
    clutter = np.zeros(new_grid.shape, dtype=np.int)
    possible_contamination = new_grid < -12.0
    clutter[possible_contamination] = 1

    z = radar.gate_altitude['data']
    clutter[(z - z.min()) > 2000.] = 0

    clutter_field = {'data': clutter,
                     'standard_name': 'clutter_mask',
                     'long_name': 'Clutter mask',
                     'comment': '0 is good, 1 is clutter',
                     'valid_min': 0,
                     'valid_max': 1,
                     'units': 'unitless'}

    return clutter_field
 
file = '/Users/sharm261/Desktop/mount/May_19_2013_all_stuff/KTLX_data/KTLX20130519_212339_V06'

radar = pyart.io.read(file)

norm_coherent_power=copy.deepcopy(radar.fields['differential_phase'])
norm_coherent_power['data'] = norm_coherent_power['data']*0.+1.
radar.fields['normalized_coherent_power'] = norm_coherent_power

sounding = pd.read_csv('/Users/sharm261/Desktop/MPEX_soundings/research.Purdue_sonde.20130519205303.skewT.txt',delimiter='\t')

sounding.columns = ['Time','Temp (C)', 'vir T', 'RH', 'Pressure (hPa)', 'geo hght', 'RDF elev','RDF azimuth' ,'wind dir', 
                    'wind speed (kts)','slant range','slant flags' ,'lat','lon','raw temp','geometric height (m)', 'rec time (s)','hght stat']

hght = np.array(sounding['geometric height (m)']).squeeze()
temp = np.array(sounding['Temp (C)']).squeeze()

z_dict, temp_dict = pyart.retrieve.map_profile_to_gates(temp, hght, radar)

radar.add_field('temp',temp_dict)

snr = pyart.retrieve.calculate_snr_from_reflectivity(radar)
velocity_texture = pyart.retrieve.calculate_velocity_texture(radar)
phidp_texture = pyart.retrieve.texture_of_complex_phase(radar)

radar.add_field('sounding_temperature', temp_dict, replace_existing=True)
radar.add_field('height', z_dict, replace_existing=True)
radar.add_field('signal_to_noise_ratio', snr, replace_existing=True)
radar.add_field('velocity_texture', velocity_texture, replace_existing=True)
radar.add_field('differential_phase_texture',phidp_texture,replace_existing=True)

do_my_fuzz(radar,rhv_field='cross_correlation_ratio',ncp_field='normalized_coherent_power')

my_fuzz, _ = do_my_fuzz(radar,rhv_field='cross_correlation_ratio',ncp_field='normalized_coherent_power')

radar.add_field('gate_id', my_fuzz,replace_existing=True)

filter = pyart.filters.GateFilter(radar)
filter.exclude_below('reflectivity', 20)
filter = pyart.correct.despeckle_field(radar, 'reflectivity', size=30, gatefilter=filter)
corrected_reflectivity = copy.deepcopy(radar.fields['reflectivity'])
corrected_reflectivity['data'] = np.ma.masked_where(filter.gate_excluded, corrected_reflectivity['data'])
radar.add_field('corrected_reflectivity', corrected_reflectivity, replace_existing=True)

new_clutter_field = gen_clutter_field_from_refl(radar, 'corrected_reflectivity','reflectivity')
radar.add_field('ground_clutter', new_clutter_field, replace_existing=True)

if 'ground_clutter' in radar.fields.keys():
        # Adding fifth gate id, clutter.
        clutter_data = radar.fields['ground_clutter']['data']
        gate_data = radar.fields['gate_id']['data']
        radar.fields['gate_id']['data'][clutter_data == 1] = 5
        notes = radar.fields['gate_id']['notes']
        radar.fields['gate_id']['notes'] = notes + ',5:clutter'
        radar.fields['gate_id']['valid_max'] = 5
cat_dict = {}
for pair_str in radar.fields['gate_id']['notes'].split(','):
    cat_dict.update({pair_str.split(':')[1]:int(pair_str.split(':')[0])})

cmac_gates = pyart.correct.GateFilter(radar)
cmac_gates.exclude_all()
cmac_gates.include_equal('gate_id', cat_dict['rain'])
cmac_gates.include_equal('gate_id', cat_dict['melting'])
cmac_gates.include_equal('gate_id', cat_dict['snow'])

from cmac.cmac_processing import fix_phase_fields

phidp, kdp = pyart.correct.phase_proc_lp_gf(
        radar, gatefilter=cmac_gates, debug=True,
        nowrap=50, fzl=15000,coef=0.914,low_z=25)
phidp_filt, kdp_filt = fix_phase_fields(
    copy.deepcopy(kdp), copy.deepcopy(phidp), radar.range['data'],
    cmac_gates)

radar.add_field('corrected_differential_phase', phidp,
                replace_existing=True)
radar.add_field('filtered_corrected_differential_phase', phidp_filt,
                replace_existing=True)
radar.add_field('corrected_specific_diff_phase', kdp,
                replace_existing=True)
radar.add_field('filtered_corrected_specific_diff_phase', kdp_filt,
                replace_existing=True)

radar.add_field('kdp',kdp,replace_existing=True)

display = pyart.graph.RadarMapDisplay(radar)
display.plot_ppi_map('kdp',0)

xsect = pyart.util.cross_section_ppi(radar, [350,345])

xmax = 140
fzl = 4.2

display = pyart.graph.RadarDisplay(xsect)
fig = plt.figure(figsize=(13,16))
ax = fig.add_subplot(411)
display.plot('reflectivity', 0, vmin=0, vmax=70.,cmap='pyart_NWSRef')
ax.set_ylim(0.,15)
ax.set_xlim(0.,xmax)
ax = fig.add_subplot(412)
display.plot('reflectivity', 1, vmin=0, vmax=70.,cmap='pyart_NWSRef')
ax.set_ylim(0.,15)
ax.set_xlim(0.,xmax)

ax = fig.add_subplot(413)
display.plot('kdp', 0, vmin=-.5, vmax=3.,cmap=cm.gist_stern)
ax.plot([0.,xmax],[fzl,fzl],'w--')
ax.set_ylim(0.,15)
ax.set_xlim(0.,xmax)
ax = fig.add_subplot(414)
display.plot('kdp', 1, vmin=-.5, vmax=3.,cmap=cm.gist_stern)
ax.plot([0.,xmax],[fzl,fzl],'w--')
ax.set_ylim(0.,15)
ax.set_xlim(0.,xmax)

plt.tight_layout()
plt.show()