get_thredds_nc.py

import os, sys
from glob import glob
from IPython import embed
from subprocess import call
# example <a href='catalog.html?dataset=NOAA/CBOFS/MODELS/201710/nos.cbofs.stations.nowcast.20171031.t18z.nc'><tt>nos.cbofs.stations.nowcast.20171031.t18z.nc</tt></a></td>^M
fpaths = glob('/localdata/jhansen/thredds/opendap.co-ops.nos.noaa.gov/thredds/catalog/NOAA/*')
output = '/localdata/jhansen/thredds/ncfiles'
fileServer = 'https://opendap.co-ops.nos.noaa.gov/thredds/fileServer/'
for station in fpaths:
    # only get stations which have fairly open oceans
    if os.path.split(station)[1] in ['TBOFS','NGOFS', 'GOMOFS', 'DBOFS', 'CREOFS', 'CBOFS']:
        print("Getting %s" %station)
        station_dates = glob(os.path.join(station, 'MODELS/*'))
        print(station_dates)
        for d in station_dates:
            cat = (os.path.join(d, 'catalog.html'))
            if os.path.exists(cat):
                ll = open(cat, 'r').readlines()
                for l in ll:
                    if '.nc' in l:
                        le = l.index('.nc')
                        ls = l.index('dataset=')
                        ncpath = l[ls+len('dataset='):le+len('.nc')]
                        p = fileServer+ncpath