HRRR-check.ipynb

HRRR-check.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import pandas as pd\n\n\ndates = pd.date_range(\n    start=\"2019-01-01 00:00\",\n    end=\"2019-12-31 23:00\",\n    freq=\"1h\"\n)",
      "execution_count": 1,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "import fsspec\n\n\nfs = fsspec.filesystem(\"s3\", anon=True)",
      "execution_count": 2,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "def hrrr_aws(date):\n    status = True\n    yyyymmdd = date.strftime('%Y%m%d')\n    hh = date.strftime('%H')\n    cfile = f's3://noaa-hrrr-bdp-pds/hrrr.{yyyymmdd}/conus/hrrr.t{hh}z.wrfsfcf01.grib2'\n    flist = fs.ls(cfile)\n    if not flist:\n        status = False\n    return status",
      "execution_count": 3,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "hrrr_aws(dates[0])",
      "execution_count": 4,
      "outputs": [
        {
          "data": {
            "text/plain": "True"
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "nsub = 10  # small number to test",
      "execution_count": 5,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "%%time\nret = [hrrr_aws(date) for date in dates[:nsub]]\nprint(ret)",
      "execution_count": 6,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "[True, True, True, True, True, True, True, True, True, True]\nCPU times: user 91.5 ms, sys: 3.82 ms, total: 95.3 ms\nWall time: 1.86 s\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "from joblib import Parallel, delayed\nimport multiprocessing\n\n\nnum_cores = multiprocessing.cpu_count()\nprint(num_cores)",
      "execution_count": 7,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "8\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "%%time\n\n\nstatus = Parallel(n_jobs=1)(delayed(hrrr_aws)(date) for date in dates[:nsub]\n)\nprint(status)",
      "execution_count": 8,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "[True, True, True, True, True, True, True, True, True, True]\nCPU times: user 20.9 ms, sys: 4.26 ms, total: 25.2 ms\nWall time: 22.8 ms\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "%%time\n\n\nstatus = Parallel(n_jobs=2)(delayed(hrrr_aws)(date) for date in dates[:nsub]\n)\nprint(status)",
      "execution_count": 9,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "[True, True, True, True, True, True, True, True, True, True]\nCPU times: user 54.1 ms, sys: 23.3 ms, total: 77.4 ms\nWall time: 6.89 s\n"
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.6",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "",
      "data": {
        "description": "HRRR-check.ipynb",
        "public": true
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}