Skip to content

Instantly share code, notes, and snippets.

@kaedonkers

kaedonkers/pp_to_zarr_hourly.ipynb Secret

Last active June 4, 2021 05:43
Show Gist options
  • Save kaedonkers/028a70aa439ae7b0d4d9e2429990eb3b to your computer and use it in GitHub Desktop.
Save kaedonkers/028a70aa439ae7b0d4d9e2429990eb3b to your computer and use it in GitHub Desktop.
Download ZIP
Jupyter Notebook example of converting hundreds of PP files into one Zarr
Display the source blob
Display the rendered blob
Raw
pp_to_zarr_hourly.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Create Zarr of 9 years of hourly CSSP China data [1851-1859]"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import iris\n",
"import os\n",
"import sys\n",
"import logging\n",
"import xarray as xr\n",
"import numpy as np\n",
"\n",
"import crd_utils as crd\n",
"import umdates_utils as um\n",
"\n",
"from datetime import datetime, timedelta"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create a list of all the files we want to process"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"5850\n"
]
}
],
"source": [
"# hourly data filenames\n",
"filepath = '/data/cssp-china/pp_dataset/hourly'\n",
"files = sorted(os.listdir(filepath))\n",
"print(len(files))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"3288\n"
]
}
],
"source": [
"# generate all possible filenames for the time period\n",
"runid = 'apepd'\n",
"startd = datetime(1851, 1, 1) # 00Z on Jan 01 1851\n",
"endd = datetime(1860, 1, 1) # 00Z on Jan 01 1860\n",
"freq = 'pj'\n",
"\n",
"decade_filenames = um.UMFileList(runid, startd, endd, freq)\n",
"print(len(decade_filenames))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"329\n"
]
}
],
"source": [
"# find the overlap of actual filenames with all possible filenames in that decade\n",
"filenames = list(set(files).intersection(set(decade_filenames)))\n",
"filenames.sort()\n",
"print(len(filenames))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"329\n"
]
}
],
"source": [
"filepaths = [os.path.join(filepath, filename) for filename in filenames]\n",
"print(len(filepaths))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Set all the Cube, Dataset and Zarr variables we need to process the cubes"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 1.19 s, sys: 79.6 ms, total: 1.27 s\n",
"Wall time: 2.13 s\n"
]
},
{
"data": {
"text/html": [
"\n",
"<style>\n",
" a.iris {\n",
" text-decoration: none !important;\n",
" }\n",
" table.iris {\n",
" white-space: pre;\n",
" border: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-family: monaco, monospace;\n",
" }\n",
" th.iris {\n",
" background: #303f3f;\n",
" color: #e0e0e0;\n",
" border-left: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-size: 1.05em;\n",
" min-width: 50px;\n",
" max-width: 125px;\n",
" }\n",
" tr.iris :first-child {\n",
" border-right: 1px solid #9c9c9c !important;\n",
" }\n",
" td.iris-title {\n",
" background: #d5dcdf;\n",
" border-top: 1px solid #9c9c9c;\n",
" font-weight: bold;\n",
" }\n",
" .iris-word-cell {\n",
" text-align: left !important;\n",
" white-space: pre;\n",
" }\n",
" .iris-subheading-cell {\n",
" padding-left: 2em !important;\n",
" }\n",
" .iris-inclusion-cell {\n",
" padding-right: 1em !important;\n",
" }\n",
" .iris-panel-body {\n",
" padding-top: 0px;\n",
" }\n",
" .iris-panel-title {\n",
" padding-left: 3em;\n",
" }\n",
" .iris-panel-title {\n",
" margin-top: 7px;\n",
" }\n",
"</style>\n",
"<table class=\"iris\" id=\"140704819325192\">\n",
" <tr class=\"iris\">\n",
"<th class=\"iris iris-word-cell\">Surface Air Pressure (Pa)</th>\n",
"<th class=\"iris iris-word-cell\">time</th>\n",
"<th class=\"iris iris-word-cell\">grid_latitude</th>\n",
"<th class=\"iris iris-word-cell\">grid_longitude</th>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
"<td class=\"iris-word-cell iris-subheading-cell\">Shape</td>\n",
"<td class=\"iris iris-inclusion-cell\">97</td>\n",
"<td class=\"iris iris-inclusion-cell\">219</td>\n",
"<td class=\"iris iris-inclusion-cell\">286</td>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Dimension coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\ttime</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_latitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_longitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Auxiliary coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_period</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Scalar coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_reference_time</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">1849-12-01 00:00:00</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Attributes</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tSTASH</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">m01s00i001</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tsource</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">Data from Met Office Unified Model</td>\n",
"</tr>\n",
"</table>\n",
" "
],
"text/plain": [
"<iris 'Cube' of surface_air_pressure / (Pa) (time: 97; grid_latitude: 219; grid_longitude: 286)>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"cubelist = iris.load(filepaths[0:1])\n",
"cubelist[0]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0: surface_air_pressure / (Pa) (time: 97; grid_latitude: 219; grid_longitude: 286)\n",
"1: x_wind / (m s-1) (time: 97; grid_latitude: 218; grid_longitude: 286)\n",
"2: y_wind / (m s-1) (time: 97; grid_latitude: 218; grid_longitude: 286)\n"
]
}
],
"source": [
"print(cubelist)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"CUBENAMES = [cube.name() for cube in cubelist]\n",
"UNIQUE_COORDS = crd.unique_coords_list(cubelist)\n",
"COORD_NAME_MAPPING = crd.get_new_coord_names(UNIQUE_COORDS)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['time', 'grid_latitude', 'grid_longitude', 'forecast_reference_time', 'forecast_period', 'grid_latitude', 'grid_longitude', 'height']\n"
]
}
],
"source": [
"print([coord.name() for coord in UNIQUE_COORDS])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('grid_latitude_1', 'grid_longitude_1')"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"COORD_NAME_MAPPING[1]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"ZARR = '/data/cssp-china/zarr_hourly_1851-1859'\n",
"CHUNKS = {'time': 200, 'grid_latitude': 219, 'grid_longitude': 286, 'grid_latitude_1': 218, 'grid_longitude_1': 286}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Initialise logging"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"LOGFILE = '/data/cssp-china/zarr_append_hourly.log'\n",
"logging.basicConfig(filename=LOGFILE,\n",
" level=logging.DEBUG,\n",
" format='%(asctime)s %(message)s',\n",
" datefmt='%d/%m/%Y %H:%M:%S')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"if not os.path.isfile(LOGFILE):\n",
" os.mknod(LOGFILE)\n",
"\n",
"if os.stat(LOGFILE).st_size == 0:\n",
" logging.info('Initiate log')\n",
" print(f'Log initiated at {LOGFILE}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create a Zarr with 3 cubes, which definitely amounts to 577 time steps (~25 days)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 6.92 s, sys: 376 ms, total: 7.3 s\n",
"Wall time: 8.42 s\n"
]
},
{
"data": {
"text/html": [
"\n",
"<style>\n",
" a.iris {\n",
" text-decoration: none !important;\n",
" }\n",
" table.iris {\n",
" white-space: pre;\n",
" border: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-family: monaco, monospace;\n",
" }\n",
" th.iris {\n",
" background: #303f3f;\n",
" color: #e0e0e0;\n",
" border-left: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-size: 1.05em;\n",
" min-width: 50px;\n",
" max-width: 125px;\n",
" }\n",
" tr.iris :first-child {\n",
" border-right: 1px solid #9c9c9c !important;\n",
" }\n",
" td.iris-title {\n",
" background: #d5dcdf;\n",
" border-top: 1px solid #9c9c9c;\n",
" font-weight: bold;\n",
" }\n",
" .iris-word-cell {\n",
" text-align: left !important;\n",
" white-space: pre;\n",
" }\n",
" .iris-subheading-cell {\n",
" padding-left: 2em !important;\n",
" }\n",
" .iris-inclusion-cell {\n",
" padding-right: 1em !important;\n",
" }\n",
" .iris-panel-body {\n",
" padding-top: 0px;\n",
" }\n",
" .iris-panel-title {\n",
" padding-left: 3em;\n",
" }\n",
" .iris-panel-title {\n",
" margin-top: 7px;\n",
" }\n",
"</style>\n",
"<table class=\"iris\" id=\"139637513800504\">\n",
" <tr class=\"iris\">\n",
"<th class=\"iris iris-word-cell\">X Wind (m s-1)</th>\n",
"<th class=\"iris iris-word-cell\">time</th>\n",
"<th class=\"iris iris-word-cell\">grid_latitude</th>\n",
"<th class=\"iris iris-word-cell\">grid_longitude</th>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
"<td class=\"iris-word-cell iris-subheading-cell\">Shape</td>\n",
"<td class=\"iris iris-inclusion-cell\">577</td>\n",
"<td class=\"iris iris-inclusion-cell\">218</td>\n",
"<td class=\"iris iris-inclusion-cell\">286</td>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Dimension coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\ttime</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_latitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_longitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Auxiliary coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_period</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Scalar coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_reference_time</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">1849-12-01 00:00:00</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\theight</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">10.0 m</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Attributes</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tSTASH</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">m01s03i225</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tsource</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">Data from Met Office Unified Model</td>\n",
"</tr>\n",
"</table>\n",
" "
],
"text/plain": [
"<iris 'Cube' of x_wind / (m s-1) (time: 577; grid_latitude: 218; grid_longitude: 286)>"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"cubelist0 = iris.load(filepaths[0:3])\n",
"cubelist0[1]"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"surface_air_pressure\n",
" x time\n",
" x grid_latitude\n",
" x grid_longitude\n",
" x forecast_reference_time\n",
" x forecast_period\n",
"x_wind\n",
" x time\n",
" grid_latitude_1\n",
" grid_longitude_1\n",
" x forecast_reference_time\n",
" x height\n",
" x forecast_period\n",
"y_wind\n",
" x time\n",
" grid_latitude_1\n",
" grid_longitude_1\n",
" x forecast_reference_time\n",
" x height\n",
" x forecast_period\n",
"CPU times: user 3.67 ms, sys: 0 ns, total: 3.67 ms\n",
"Wall time: 2.02 ms\n"
]
}
],
"source": [
"%%time\n",
"crd.rename_cubes(cubelist0, CUBENAMES, COORD_NAME_MAPPING, dryrun=False)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 35.5 ms, sys: 3.96 ms, total: 39.5 ms\n",
"Wall time: 52.2 ms\n"
]
}
],
"source": [
"%%time\n",
"dalist0 = crd.cubelist_to_dalist(cubelist0)\n",
"ds0 = xr.merge(dalist0)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre>&lt;xarray.Dataset&gt;\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 577)\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1851-01-25\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" forecast_reference_time datetime64[ns] 1849-12-01\n",
" forecast_period (time) timedelta64[ns] 396 days 00:00:00 ... 420 days 00:00:00\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 10.0\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array&lt;chunksize=(1, 219, 286), meta=np.ndarray&gt;\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(1, 218, 286), meta=np.ndarray&gt;\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(1, 218, 286), meta=np.ndarray&gt;</pre>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 577)\n",
"Coordinates:\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1851-01-25\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" forecast_reference_time datetime64[ns] 1849-12-01\n",
" forecast_period (time) timedelta64[ns] 396 days 00:00:00 ... 420 days 00:00:00\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 10.0\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array<chunksize=(1, 219, 286), meta=np.ndarray>\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(1, 218, 286), meta=np.ndarray>\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(1, 218, 286), meta=np.ndarray>"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds0"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 4.49 s, sys: 1.27 s, total: 5.77 s\n",
"Wall time: 17.6 s\n"
]
}
],
"source": [
"%%time\n",
"logging.info(f'Creating {ZARR}')\n",
"crd.ds_to_zarr(ds0, ZARR, chunks=CHUNKS)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre>&lt;xarray.Dataset&gt;\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 577)\n",
"Coordinates:\n",
" forecast_period (time) timedelta64[ns] dask.array&lt;chunksize=(577,), meta=np.ndarray&gt;\n",
" forecast_reference_time datetime64[ns] ...\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 ...\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1851-01-25\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array&lt;chunksize=(200, 219, 286), meta=np.ndarray&gt;\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(200, 218, 286), meta=np.ndarray&gt;\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(200, 218, 286), meta=np.ndarray&gt;</pre>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 577)\n",
"Coordinates:\n",
" forecast_period (time) timedelta64[ns] dask.array<chunksize=(577,), meta=np.ndarray>\n",
" forecast_reference_time datetime64[ns] ...\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 ...\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1851-01-25\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array<chunksize=(200, 219, 286), meta=np.ndarray>\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(200, 218, 286), meta=np.ndarray>\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(200, 218, 286), meta=np.ndarray>"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dsz0 = xr.open_zarr(ZARR)\n",
"dsz0"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.432370924"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Size of Dataset in GB\n",
"dsz0.nbytes / 1e9"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"logging.info(f' Successfull creation of {ZARR}')\n",
"logging.info(f' Processed filenames 0:{filenames[0]} - 3:{filenames[3]}')\n",
"logging.info(f' Chunking {CHUNKS}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Now loop through the remaining cubes and append to the Zarr we created"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"def logprint(message):\n",
" logging.info(message)\n",
" print(message)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Appending data to /data/cssp-china/zarr_hourly_1851-1859\n",
" Processing filenames 207:apepda.pj56950.pp - 209:apepda.pj569f0.pp\n",
" | Loaded files successfully\n",
" | Renamed files successfully\n",
" | Created dataset successfully\n",
" | Appended to Zarr /data/cssp-china/zarr_hourly_1851-1859 successfully\n",
" Processing filenames 209:apepda.pj569p0.pp - 211:apepda.pj56a50.pp\n",
" \n",
" ...\n",
" \n",
" Processing filenames 327:apepda.pj59cj0.pp - 329:apepda.pj59ct0.pp\n",
" | Loaded files successfully\n",
" | Renamed files successfully\n",
" | Created dataset successfully\n",
" | Appended to Zarr /data/cssp-china/zarr_hourly_1851-1859 successfully\n",
" Appending data complete\n",
"CPU times: user 10min 5s, sys: 1min 25s, total: 11min 30s\n",
"Wall time: 21min 47s\n"
]
}
],
"source": [
"%%time\n",
"# Loop through in bunches of 2 cubes, so as to not use to much memory at one time\n",
"start = 207\n",
"step = 2\n",
"stop = len(filepaths)\n",
"# stop = start+(2*step)\n",
"\n",
"logprint(f'Appending data to {ZARR}')\n",
"\n",
"for i in range(start, stop, step):\n",
" fnames = filenames[i:i+step]\n",
" logprint(f' Processing filenames {i}:{fnames[0]} - {min([i+step, stop])}:{fnames[-1]}')\n",
" try:\n",
" cubelist = iris.load(filepaths[i:i+step])\n",
" logprint(f' | Loaded files successfully')\n",
" \n",
" crd.rename_cubes(cubelist, CUBENAMES, COORD_NAME_MAPPING, dryrun=False, verbose=False)\n",
" logprint(f' | Renamed files successfully')\n",
" \n",
" dalist = crd.cubelist_to_dalist(cubelist)\n",
" ds = xr.merge(dalist)\n",
" logprint(f' | Created dataset successfully')\n",
" \n",
" crd.ds_to_zarr(ds, ZARR, chunks=CHUNKS)\n",
" logprint(f' | Appended to Zarr {ZARR} successfully')\n",
" \n",
" except Exception as e:\n",
" logprint(f' X ERROR: {e}')\n",
" raise e\n",
"\n",
"logprint(f' Appending data complete')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Let's open the Zarr we have appended to and check it"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<pre>&lt;xarray.Dataset&gt;\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 77329)\n",
"Coordinates:\n",
" forecast_period (time) timedelta64[ns] dask.array&lt;chunksize=(577,), meta=np.ndarray&gt;\n",
" forecast_reference_time datetime64[ns] ...\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 ...\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1859-12-29\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array&lt;chunksize=(200, 219, 286), meta=np.ndarray&gt;\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(200, 218, 286), meta=np.ndarray&gt;\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array&lt;chunksize=(200, 218, 286), meta=np.ndarray&gt;</pre>"
],
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (grid_latitude: 219, grid_latitude_1: 218, grid_longitude: 286, grid_longitude_1: 286, time: 77329)\n",
"Coordinates:\n",
" forecast_period (time) timedelta64[ns] dask.array<chunksize=(577,), meta=np.ndarray>\n",
" forecast_reference_time datetime64[ns] ...\n",
" * grid_latitude (grid_latitude) float32 22.88 22.66 ... -25.08\n",
" * grid_latitude_1 (grid_latitude_1) float32 22.77 ... -24.969997\n",
" * grid_longitude (grid_longitude) float32 323.48 323.7 ... 386.18002\n",
" * grid_longitude_1 (grid_longitude_1) float32 323.59003 ... 386.29004\n",
" height float64 ...\n",
" * time (time) datetime64[ns] 1851-01-01 ... 1859-12-29\n",
"Data variables:\n",
" surface_air_pressure (time, grid_latitude, grid_longitude) float32 dask.array<chunksize=(200, 219, 286), meta=np.ndarray>\n",
" x_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(200, 218, 286), meta=np.ndarray>\n",
" y_wind (time, grid_latitude_1, grid_longitude_1) float32 dask.array<chunksize=(200, 218, 286), meta=np.ndarray>"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dsz1 = xr.open_zarr(ZARR)\n",
"dsz1"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"<style>\n",
" a.iris {\n",
" text-decoration: none !important;\n",
" }\n",
" table.iris {\n",
" white-space: pre;\n",
" border: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-family: monaco, monospace;\n",
" }\n",
" th.iris {\n",
" background: #303f3f;\n",
" color: #e0e0e0;\n",
" border-left: 1px solid;\n",
" border-color: #9c9c9c;\n",
" font-size: 1.05em;\n",
" min-width: 50px;\n",
" max-width: 125px;\n",
" }\n",
" tr.iris :first-child {\n",
" border-right: 1px solid #9c9c9c !important;\n",
" }\n",
" td.iris-title {\n",
" background: #d5dcdf;\n",
" border-top: 1px solid #9c9c9c;\n",
" font-weight: bold;\n",
" }\n",
" .iris-word-cell {\n",
" text-align: left !important;\n",
" white-space: pre;\n",
" }\n",
" .iris-subheading-cell {\n",
" padding-left: 2em !important;\n",
" }\n",
" .iris-inclusion-cell {\n",
" padding-right: 1em !important;\n",
" }\n",
" .iris-panel-body {\n",
" padding-top: 0px;\n",
" }\n",
" .iris-panel-title {\n",
" padding-left: 3em;\n",
" }\n",
" .iris-panel-title {\n",
" margin-top: 7px;\n",
" }\n",
"</style>\n",
"<table class=\"iris\" id=\"140704808257912\">\n",
" <tr class=\"iris\">\n",
"<th class=\"iris iris-word-cell\">Surface Air Pressure (Pa)</th>\n",
"<th class=\"iris iris-word-cell\">time</th>\n",
"<th class=\"iris iris-word-cell\">grid_latitude</th>\n",
"<th class=\"iris iris-word-cell\">grid_longitude</th>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
"<td class=\"iris-word-cell iris-subheading-cell\">Shape</td>\n",
"<td class=\"iris iris-inclusion-cell\">77329</td>\n",
"<td class=\"iris iris-inclusion-cell\">219</td>\n",
"<td class=\"iris iris-inclusion-cell\">286</td>\n",
"</tr>\n",
" <tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Dimension coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\ttime</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_latitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tgrid_longitude</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Auxiliary coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_period</td>\n",
" <td class=\"iris-inclusion-cell\">x</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
" <td class=\"iris-inclusion-cell\">-</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Scalar coordinates</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tforecast_reference_time</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">1849-12-01 00:00:00</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\theight</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">10.0 m</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-title iris-word-cell\">Attributes</td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
" <td class=\"iris-title\"></td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tSTASH</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">[1, 0, 1]</td>\n",
"</tr>\n",
"<tr class=\"iris\">\n",
" <td class=\"iris-word-cell iris-subheading-cell\">\tsource</td>\n",
" <td class=\"iris-word-cell\" colspan=\"3\">Data from Met Office Unified Model</td>\n",
"</tr>\n",
"</table>\n",
" "
],
"text/plain": [
"<iris 'Cube' of surface_air_pressure / (Pa) (time: 77329; grid_latitude: 219; grid_longitude: 286)>"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cubez = dsz1.surface_air_pressure.to_iris()\n",
"cubez"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Is the time coordinate contiguous?"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([1, 1, 1, ..., 1, 1, 1])"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"deltas = cubez.coord('time').points[1:]-cubez.coord('time').points[0:-1]\n",
"deltas"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"def plot_td_hist(td, **kwargs):\n",
" plt.hist(td, log=True, **kwargs)\n",
" plt.gcf().set_size_inches(15, 5)\n",
"# plt.xticks(np.arange(1, 31))\n",
" plt.xlabel('Hours')\n",
" plt.ylabel('Number of timedeltas')\n",
" plt.title(f'Timedeltas for hourly data')\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAAFNCAYAAABfWL0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3X+4ZWVd9/H3xxlQQx0w0OTHMOgQV5OW1gkzS7G0BnVA8UeMVorkaJcopk+J5uOPfEos6VHDtDEIEgURFRkdw/IJKbMETOWX6IgQIyjgjwEUxYHv88deRzenOeesM7PX7NnrvF/Xda6z1732utf3nLOuPXy473WvVBWSJEmSpP66x7gLkCRJkiR1y+AnSZIkST1n8JMkSZKknjP4SZIkSVLPGfwkSZIkqecMfpIkSZLUcwY/SdKskrwqyd/tpHNdkOT3W763kqzcCTU9Ncl1SW5L8ogR9Nf6ZxzBuQ5LsnlEfZ2W5P+Moi9J0ngsHXcBkqTxSXLb0OZPAD8A7my2X1BVf77zq1qYJKcBm6vq1R10/2bguKr6cAd991KSC4Azqmqn/A8DSVI7Bj9JWsSq6j7Tr5NcA/x+Vf3z+Cra5RwIXL49ByZZUlV3zv/O0Uviv++SpLtxqqckaVZJXpfkjOb1imaK5THN9MdvJ3lhkl9K8oUk30ly8ozjn5fkyua95yc5cGjfE5J8McmW5ri0PXboPeuAZwN/3EzH3NC0n5DkK0luTXJFkqcOHbMyySeb896c5H3b6PeezWjoEuDzSb7StP9MM13zO0kuT3LE0DGnJXlHko1Jvgs8bpZf64FJPtXU9vEkew/1cUTT73ea8/zM0L67TW8dnn45Pa0zySuSfB34+xk/zx8l+cCMtr9O8pZtFZjkEUk+29T4PuBeQ/v2SvKRJDc1f5uPJNm/2fdnwK8BJzd/j5Ob9rc218wtSS5J8muz/G4kSR0x+EmSFuqRwMHAbwNvAf4EeDzws8AzkzwWIMlTgFcBRwH7AP8KnNns2xv4APBqYG/gK8Cjp08w17HDqmo98B7gL6rqPlW1ptn1FQYBZBnweuCMJA9q9r0B+DiwF7A/8Nfb6PcHQ6OhP19VD0myG7ChOfYBwIuB9yQ5ZOjQZwF/BtwX+LdZfn/PAo5p+tgd+F/Nz/zTzc/40uZn3ghsSLL7LP3M9FPA/RmMUq6bse8MYHWSPZtzLWXw93v3zE6a853b7Ls/8H7gaUNvuQeDYHkgsBy4HTgZoKr+hMHf6rjm73Fcc8xFwMOb/t4LvD/JvZAk7TQGP0nSQr2hqr5fVR8HvgucWVU3VtXXGPxH//QiKC8A3lhVV1bVVuDPgYc3I3dPBK6oqnOq6ocMAuTXh84x17Hzqqr3V9X1VXVXVb0P+DJwaLP7hwxCy77NzzFbQJvpl4H7ACdW1R1V9f+AjwBrh97z4ar6VHPe78/Sz99X1Zeq6nbgbAaBCAZB7KNV9U/N7+TNwL2BX2lZ313Aa5vQevvwjqq6AbgQeEbTtBq4uaoumeXn3A14S1X9sKrOYRDcpvv6ZlV9oKq+V1W3Mgi6j52rsKo6ozlua1WdBNwTOGSuYyRJo2XwkyQt1DeGXt++je3pkbIDgbc20xa/A3yLwXTO/YB9geumD6qqGt6e59h5Jfm9JJ8bOv6hDEYWAf646eszzbTK57Xpc7rmqrprqO3aGTVdx/yGA+73+PHva9+mPwCa81xHy58ZuGmOsAlwOvA7zevfYRujfUN1fK35m0z7UV1JfiLJ3ya5NsktDALlnkmWzHbiJC9vpu1uaf4ey/jx30OStBMY/CRJXbmOwcqgew593buq/h24AThg+o1JMrw9z7EzDQcUmlHBdwHHAT9ZVXsCl9HcQ1hVX6+q51fVvgxGFv8m7R4NcT1wQJLhfzuXA1+brZYFup5B4J3+OaZ/J9P9f4/ByqvTfmrG8fOd+1zg55I8FHgygymy23IDsF9z/mnLh16/nMFo3SOr6n7AY6ZL3lYdzf18rwCeCezV/D22DL1fkrQTGPwkSV15J/DKJD8LkGRZkumphh8FfjbJUc39Zi/h7kFmrmNn+gbw4KHtPRiEj5uaY49hMOJHs/2M6cVIgG83722z+uZ/Mpja+sdJdktyGLAGOKvFsW2cDTwpyW809xO+nMHjNabD7ueAZyVZkmQ180yvnKkZDTyHwT12n6mq/57lrZ8GtgIvSbI0yVH8eJosDO5fvB34TpL7A6+dcfzMv8d9m/5uApYmeQ1wv4XULknacQY/SVInqupDwJuAs5opgZcBhzf7bmZwv9mJwDcZLBbzqTbHbsMpwKpmWue5VXUFcBKDAPMN4GHDfQO/BPxns2rnecDxVfXVFj/PHcARTR03A38D/F5VfbHFr2NeVXUVgymYf930vwZY05wX4Pim7TsMVjI9dztOczqD38ds0zynf86jgOcyCMa/DXxw6C1vYXDv4c3AfwD/OKOLtwJPb1b8fBtwPvAx4EsMpox+n3ZTYiVJI5S7T+GXJEl9lWQ58EXgp6rqlnHXI0naeRzxkyRpEWjuTXwZcJahT5IWn6XjLkCSJHUryR4Mpr1ey+BRDpKkRcapnpIkSZLUc071lCRJkqSeM/hJkiRJUs9N9D1+e++9d61YsWLcZUiSJEnSWFxyySU3V9U+871vooPfihUruPjii8ddhiRJkiSNRZJr27zPqZ6SJEmS1HO7TPBLcliSf03yziSHjbseSZIkSeqLToNfklOT3Jjkshntq5NclWRTkhOa5gJuA+4FbO6yLkmSJElaTLoe8TuNGQ+KTbIEeDtwOLAKWJtkFfCvVXU48Arg9R3XJUmSJEmLRqfBr6ouBL41o/lQYFNVXV1VdwBnAUdW1V3N/m8D9+yyLkmSJElaTMaxqud+wHVD25uBRyY5CvgtYE/g5NkOTrIOWAewfPnyDsuUJEmSpH4YR/DLNtqqqj4IfHC+g6tqPbAeYGpqqkZcmyRJkiT1zjhW9dwMHDC0vT9w/UI6SLImyfotW7aMtDBJkiRJ6qNxBL+LgIOTHJRkd+Bo4LyFdFBVG6pq3bJlyzopUJIkSZL6pOvHOZwJfBo4JMnmJMdW1VbgOOB84Erg7Kq6vMs6JEmSJGkx6/Qev6paO0v7RmDj9vabZA2wZuXKldvbxU6z4oSP/uj1NSc+aYyVSJIkSVqsxjHVc4c51VOSJEmS2pvI4CdJkiRJam8ig5+rekqSJElSexMZ/JzqKUmSJEntTWTwkyRJkiS1N5HBz6mekiRJktTeRAY/p3pKkiRJUnsTGfwkSZIkSe0Z/CRJkiSp5yYy+HmPnyRJkiS1N5HBz3v8JEmSJKm9iQx+kiRJkqT2DH6SJEmS1HMGP0mSJEnquYkMfi7uIkmSJEntTWTwc3EXSZIkSWpvIoOfJEmSJKk9g58kSZIk9ZzBT5IkSZJ6zuAnSZIkST03kcHPVT0lSZIkqb2JDH6u6ilJkiRJ7U1k8JMkSZIktWfwkyRJkqSeM/hJkiRJUs8Z/CRJkiSp5wx+kiRJktRzBj9JkiRJ6jmDnyRJkiT13EQGPx/gLkmSJEntTWTw8wHukiRJktTeRAY/SZIkSVJ7Bj9JkiRJ6jmDnyRJkiT1nMFPkiRJknrO4CdJkiRJPWfwkyRJkqSeM/hJkiRJUs8Z/CRJkiSp5wx+kiRJktRzu1TwS7JHkkuSPHnctUiSJElSX3Qa/JKcmuTGJJfNaF+d5Kokm5KcMLTrFcDZXdYkSZIkSYtN1yN+pwGrhxuSLAHeDhwOrALWJlmV5PHAFcA3Oq5JkiRJkhaVpV12XlUXJlkxo/lQYFNVXQ2Q5CzgSOA+wB4MwuDtSTZW1V1d1idJkiRJi0GnwW8W+wHXDW1vBh5ZVccBJHkucPNsoS/JOmAdwPLly7utVJIkSZJ6YByLu2QbbfWjF1WnVdVHZju4qtZX1VRVTe2zzz6dFChJkiRJfTKO4LcZOGBoe3/g+jHUIUmSJEmLwjiC30XAwUkOSrI7cDRw3kI6SLImyfotW7Z0UqAkSZIk9UnXj3M4E/g0cEiSzUmOraqtwHHA+cCVwNlVdflC+q2qDVW1btmyZaMvWpIkSZJ6putVPdfO0r4R2Li9/SZZA6xZuXLl9nYhSZIkSYvGOKZ67jBH/CRJkiSpvYkMfpIkSZKk9iYy+Lm4iyRJkiS1N5HBz6mekiRJktTeRAY/SZIkSVJ7Exn8nOopSZIkSe1NZPBzqqckSZIktTeRwU+SJEmS1J7BT5IkSZJ6biKDn/f4SZIkSVJ7Exn8vMdPkiRJktqbyOAnSZIkSWrP4CdJkiRJPWfwkyRJkqSem8jg5+IukiRJktTeRAY/F3eRJEmSpPYmMvhJkiRJktoz+EmSJElSzxn8JEmSJKnnDH6SJEmS1HPzBr8kD0lyz+b1YUlekmTP7kuTJEmSJI1CmxG/DwB3JlkJnAIcBLy306rm4eMcJEmSJKm9NsHvrqraCjwVeEtV/SHwoG7LmpuPc5AkSZKk9toEvx8mWQs8B/hI07ZbdyVJkiRJkkapTfA7BngU8GdV9dUkBwFndFuWJEmSJGlUls73hqq6AnjJ0PZXgRO7LEqSJEmSNDrzBr8kBwNvBFYB95pur6oHd1iXJEmSJGlE2kz1/HvgHcBW4HHAPwDv7rIoSZIkSdLotAl+966qTwCpqmur6nXAr3dbliRJkiRpVOad6gl8P8k9gC8nOQ74GvCAbsuSJEmSJI1KmxG/lwI/wWCBl18Efgf4vS6Lmo8PcJckSZKk9toEvxVVdVtVba6qY6rqacDyrgubiw9wlyRJkqT22gS/V7ZskyRJkiTtgma9xy/J4cATgf2SvG1o1/0YrPApSZIkSZoAcy3ucj1wCXBE833arcAfdlmUJEmSJGl0Zg1+VfV54PNJzqgqR/gkSZIkaULNNdXzUqCa1/9jf1X9XHdlSZIkSZJGZa6pnk/eaVVIkiRJkjoz11TPa6dfJzkQOLiq/jnJvec6TpIkSZK0a5n3cQ5Jng+cA/xt07Q/cG6XRUmSJEmSRqfNc/xeBDwauAWgqr4MPKDLoiRJkiRJo9Mm+P2gqu6Y3kiylGbRl1FK8jNJ3pnknCR/MOr+JUmSJGmxahP8PpnkVcC9kzwBeD+woU3nSU5NcmOSy2a0r05yVZJNSU4AqKorq+qFwDOBqYX9GJIkSZKk2bQJficANwGXAi8ANgKvbtn/acDq4YYkS4C3A4cDq4C1SVY1+44A/g34RMv+JUmSJEnzmHd1zqq6C3hX87UgVXVhkhUzmg8FNlXV1QBJzgKOBK6oqvOA85J8FHjvQs8nSZIkSfqfWj3AfVt24AHu+wHXDW1vBh6Z5DDgKOCeDEYVZ6trHbAOYPny5dtZgiRJkiQtHm0e4P6i5vu7m+/PBr63A+fMNtqqqi4ALpjv4KpaD6wHmJqaGvkiM5IkSZLUN/M+wD3Jo6vq0UO7TkjyKeBPt/Ocm4EDhrb3B65fSAdJ1gBrVq5cuZ0lSJIkSdLi0WZxlz2S/Or0RpJfAfbYgXNeBByc5KAkuwNHA+ctpIOq2lBV65YtW7YDZUiSJEnS4jDv4i7AscCpSZYxuOdvC/C8Np0nORM4DNg7yWbgtVV1SpLjgPOBJcCpVXX59hQvSZIkSZpfm1U9LwF+Psn9gFTVlradV9XaWdo3MscCLvNxqqckSZIktTfvVM8kD0xyCvC+qtqSZFWSY3dCbbNyqqckSZIktdfmHr/TGEzL3LfZ/hLw0q4KkiRJkiSNVpvgt3dVnQ3cBVBVW4E7O61qHknWJFm/ZUvrWaeSJEmStGi1CX7fTfKTNA9zT/LLDBZ4GRunekqSJElSe21W9XwZg8ctPKR5ft8+wNM7rUqSJEmSNDJtVvX8bJLHAocAAa6qqh92XtkcXNVTkiRJktprs6rnEuCJwG8Avwm8OMnLui5sLk71lCRJkqT22kz13AB8H7iUZoEXSZIkSdLkaBP89q+qn+u8EkmSJElSJ9qs6vmxJL/ZeSUL4OMcJEmSJKm9NsHvP4APJbk9yS1Jbk1yS9eFzcV7/CRJkiSpvTZTPU8CHgVcWlXVcT2SJEmSpBFrM+L3ZeAyQ58kSZIkTaY2I343ABck+Rjwg+nGqvqrzqqSJEmSJI1Mm+D31eZr9+Zr7HyAuyRJkiS1N2/wq6rX74xCFqKqNgAbpqamnj/uWiRJkiRpVzdr8Evylqp6aZINwP+4v6+qjui0MkmSJEnSSMw14vfu5vubd0YhkiRJkqRuzBr8quqS5uXDq+qtw/uSHA98ssvCJEmSJEmj0eZxDs/ZRttzR1yHJEmSJKkjc93jtxZ4FnBQkvOGdt0X+GbXhc3FVT0lSZIkqb257vH7dwbP8NsbOGmo/VbgC10WNR9X9ZQkSZKk9ua6x+9a4FrgUTuvHEmSJEnSqLW5x0+SJEmSNMEMfpIkSZLUc7MGvySfaL6/aeeVI0mSJEkatbkWd3lQkscCRyQ5C8jwzqr6bKeVSZIkSZJGYq7g9xrgBGB/4K9m7Cvg17sqSpIkSZI0OnOt6nkOcE6S/11Vb9iJNUmSJEmSRmiuET8AquoNSY4AHtM0XVBVH+m2rLn5AHdJkiRJam/eVT2TvBE4Hrii+Tq+aRubqtpQVeuWLVs2zjIkSZIkaSLMO+IHPAl4eFXdBZDkdOC/gFd2WZgkSZIkaTTaPsdvz6HXDrNJkiRJ0gRpM+L3RuC/kvwLg0c6PAZH+yRJkiRpYrRZ3OXMJBcAv8Qg+L2iqr7edWGSJEmSpNFoM+JHVd0AnNdxLZIkSZKkDrS9x0+SJEmSNKEMfpIkSZLUc3MGvyT3SHLZzipGkiRJkjR6cwa/5tl9n0+yfCfVI0mSJEkasTaLuzwIuDzJZ4DvTjdW1RGjLibJUxg8MP4BwNur6uOjPockSZIkLTZtgt/rd+QESU4FngzcWFUPHWpfDbwVWAL8XVWdWFXnAucm2Qt4M2DwkyRJkqQdNO/iLlX1SeAaYLfm9UXAZxdwjtOA1cMNSZYAbwcOB1YBa5OsGnrLq5v9kiRJkqQdNG/wS/J84Bzgb5um/YBz256gqi4EvjWj+VBgU1VdXVV3AGcBR2bgTcDHqmoh4VKSJEmSNIs2j3N4EfBo4BaAqvoyg3vwdsR+wHVD25ubthcDjweenuSF2zowybokFye5+KabbtrBMiRJkiSp/9rc4/eDqrojCQBJlgK1g+fNNtqqqt4GvG2uA6tqPbAeYGpqakfrkCRJkqTeazPi98kkrwLuneQJwPuBDTt43s3AAUPb+wPX72CfkiRJkqRtaBP8TgBuAi4FXgBsZLD4yo64CDg4yUFJdgeOBs5re3CSNUnWb9myZQfLkCRJkqT+a7Oq513A6cAbGDza4fSqaj3FMsmZwKeBQ5JsTnJsVW0FjgPOB64Ezq6qy9v2WVUbqmrdsmXL2h4iSZIkSYvWvPf4JXkS8E7gKwzuzTsoyQuq6mNtTlBVa2dp38hg9HDBkqwB1qxcuXJ7DpckSZKkRaXNVM+TgMdV1WFV9VjgccD/7basuTniJ0mSJEnttQl+N1bVpqHtq4EbO6pHkiRJkjRis071THJU8/LyJBuBsxk8xuEZDBZnGRunekqSJElSe3ON+K1pvu4FfAN4LHAYgxU+9+q8sjk41VOSJEmS2pt1xK+qjtmZhUiSJEmSutFmVc+DgBcDK4bfX1VHdFeWJEmSJGlU5g1+wLnAKcAG4K5uy2nHe/wkSZIkqb02we/7VfW2zitZgKraAGyYmpp6/rhrkSRJkqRdXZvg99YkrwU+DvxgurGqPttZVZIkSZKkkWkT/B4G/C7w6/x4qmc125IkSZKkXVyb4PdU4MFVdUfXxbTlPX6SJEmS1N5cz/Gb9nlgz64LWQif4ydJkiRJ7bUZ8Xsg8MUkF3H3e/x8nIMkSZIkTYA2we+1nVchSZIkSerMvMGvqj65MwpZCO/xkyRJkqT25r3HL8mtSW5pvr6f5M4kt+yM4mbjPX6SJEmS1F6bEb/7Dm8neQpwaGcVSZIkSZJGqs2qnndTVefiM/wkSZIkaWLMO+KX5KihzXsAUwwe4C5JkiRJmgBtVvVcM/R6K3ANcGQn1UiSJEmSRq7NPX7H7IxCJEmSJEndmDX4JXnNHMdVVb2hg3pa8XEOkiRJktTeXIu7fHcbXwDHAq/ouK45+TgHSZIkSWpv1hG/qjpp+nWS+wLHA8cAZwEnzXacJEmSJGnXMuc9fknuD7wMeDZwOvALVfXtnVGYJEmSJGk05rrH7y+Bo4D1wMOq6radVpUkSZIkaWTmusfv5cC+wKuB65Pc0nzdmuSWnVOeJEmSJGlHzXWP31yhUJIkSZI0IQx3kiRJktRzBj9JkiRJ6rmJDH5J1iRZv2XLlnGXIkmSJEm7vIkMfj7AXZIkSZLam8jgJ0mSJElqz+AnSZIkST1n8JMkSZKknjP4SZIkSVLPGfwkSZIkqecMfpIkSZLUcwY/SZIkSeo5g58kSZIk9ZzBT5IkSZJ6bpcJfkkenOSUJOeMuxZJkiRJ6pNOg1+SU5PcmOSyGe2rk1yVZFOSEwCq6uqqOrbLeiRJkiRpMep6xO80YPVwQ5IlwNuBw4FVwNokqzquQ5IkSZIWrU6DX1VdCHxrRvOhwKZmhO8O4CzgyC7rkCRJkqTFbBz3+O0HXDe0vRnYL8lPJnkn8Igkr5zt4CTrklyc5OKbbrqp61olSZIkaeItHcM5s422qqpvAi+c7+CqWg+sB5iamqoR1yZJkiRJvTOOEb/NwAFD2/sD1y+kgyRrkqzfsmXLSAuTJEmSpD4aR/C7CDg4yUFJdgeOBs5bSAdVtaGq1i1btqyTAiVJkiSpT7p+nMOZwKeBQ5JsTnJsVW0FjgPOB64Ezq6qy7usQ5IkSZIWs07v8auqtbO0bwQ2bm+/SdYAa1auXLm9XUiSJEnSojGOqZ47zKmekiRJktTeRAY/SZIkSVJ7Exn8XNVTkiRJktqbyODnVE9JkiRJam8ig58kSZIkqb2JDH5O9ZQkSZKk9iYy+DnVU5IkSZLam8jgJ0mSJElqz+AnSZIkST03kcHPe/wkSZIkqb2JDH7e4ydJkiRJ7U1k8JMkSZIktWfwkyRJkqSeM/hJkiRJUs9NZPBzcRdJkiRJam8ig5+Lu0iSJElSexMZ/CRJkiRJ7Rn8JEmSJKnnDH6SJEmS1HMGP0mSJEnquYkMfq7qKUmSJEntTWTwc1VPSZIkSWpvIoOfJEmSJKk9g58kSZIk9ZzBT5IkSZJ6zuAnSZIkST1n8JMkSZKknjP4SZIkSVLPGfwkSZIkqeeWjruA7ZFkDbBm5cqV4y5FkiTpR1ac8NG7bV9z4pPGVMmubfj3NPw7mq1d0o6byBE/H+AuSZIkSe1NZPCTJEmSJLVn8JMkSZKknjP4SZIkSVLPGfwkSZIkqecMfpIkSZLUcwY/SZIkSeo5g58kSZIk9ZzBT5IkSZJ6zuAnSZIkST1n8JMkSZKknls67gKmJdkD+BvgDuCCqnrPmEuSJEmSpF7odMQvyalJbkxy2Yz21UmuSrIpyQlN81HAOVX1fOCILuuSJEmSpMWk66mepwGrhxuSLAHeDhwOrALWJlkF7A9c17ztzo7rkiRJkqRFo9OpnlV1YZIVM5oPBTZV1dUASc4CjgQ2Mwh/n2OOQJpkHbAOYPny5aMvWpIkqQMrTvjoj15fc+KTxliJJpnXkbbXOBZ32Y8fj+zBIPDtB3wQeFqSdwAbZju4qtZX1VRVTe2zzz7dVipJkiRJPTCOxV2yjbaqqu8Cx+zsYiRJkiSp78Yx4rcZOGBoe3/g+oV0kGRNkvVbtmwZaWGSJEmS1EfjCH4XAQcnOSjJ7sDRwHkL6aCqNlTVumXLlnVSoCRJkiT1SdePczgT+DRwSJLNSY6tqq3AccD5wJXA2VV1+QL7dcRPkiRJklrqelXPtbO0bwQ27kC/G4ANU1NTz9/ePiRJkiRpsRjHVE9JkiRJ0k40kcHPqZ6SJEmS1N5EBj8Xd5EkSZKk9iYy+EmSJEmS2jP4SZIkSVLPTWTw8x4/SZIkSWovVTXuGrZbkpuAa8ddxzbsDdw87iLUa15j6prXmLrk9aWueY2pa7vSNXZgVe0z35smOvjtqpJcXFVT465D/eU1pq55jalLXl/qmteYujaJ19hETvWUJEmSJLVn8JMkSZKknjP4dWP9uAtQ73mNqWteY+qS15e65jWmrk3cNeY9fpIkSZLUc474SZIkSVLPGfxGLMnqJFcl2ZTkhHHXo8mX5Joklyb5XJKLm7b7J/mnJF9uvu817jo1OZKcmuTGJJcNtc16TSV5ZfOZdlWS3xpP1Zoks1xjr0vyteaz7HNJnji0z2tMrSU5IMm/JLkyyeVJjm/a/RzTSMxxjU3055hTPUcoyRLgS8ATgM3ARcDaqrpirIVpoiW5BpiqqpuH2v4C+FZVndj8D4a9quoV46pRkyXJY4DbgH+oqoc2bdu8ppKsAs4EDgX2Bf4Z+OmqunNM5WsCzHKNvQ64rarePOO9XmNakCQPAh5UVZ9Ncl/gEuApwHPxc0wjMMc19kwm+HPMEb/ROhTYVFVXV9UdwFnAkWOuSf10JHB68/p0Bh9GUitVdSHwrRnNs11TRwJnVdUPquqrwCYGn3XSrGa5xmbjNaYFqaobquqzzetbgSuB/fBzTCMyxzU2m4m4xgx+o7UfcN3Q9mbmvkikNgr4eJJLkqxr2h5YVTfA4MMJeMDYqlNfzHZN+bmmUTouyReaqaDT0/C8xrTdkqwAHgH8J36OqQMzrjGY4M8xg99oZRttzqXVjnp0Vf0CcDjwomYKlbSz+LmmUXkH8BDg4cANwElNu9eYtkuS+wAfAF5aVbfM9dZttHmNaV7buMYm+nPM4Ddam4EDhrb3B64fUy3qiaq6vvl+I/AhBlMHvtHMP5+eh37j+CpUT8x2Tfm5ppGoqm9U1Z1VdRfwLn48DcprTAuWZDcG/0H+nqr6YNPs55hGZlvX2KR/jhn8Rusi4OAkByXZHTgaOG/MNWmCJdmjuamYJHsAvwlcxuC6ek7ztucAHx5PheqR2a6p84Cjk9wzyUHAwcBnxlCfJtz0f5A3nsrgswy8xrRASQKcAlxZVX8YZ8l1AAACU0lEQVQ1tMvPMY3EbNfYpH+OLR13AX1SVVuTHAecDywBTq2qy8dclibbA4EPDT5/WAq8t6r+MclFwNlJjgX+G3jGGGvUhElyJnAYsHeSzcBrgRPZxjVVVZcnORu4AtgKvGhXW6VMu55ZrrHDkjycwfSna4AXgNeYtsujgd8FLk3yuabtVfg5ptGZ7RpbO8mfYz7OQZIkSZJ6zqmekiRJktRzBj9JkiRJ6jmDnyRJkiT1nMFPkiRJknrO4CdJkiRJPWfwkyQJSHLbjO3nJjl5XPVIkjRKBj9JkjqUZMm4a5AkyeAnSdI8khyY5BNJvtB8X960n5bk6UPvu635fliSf0nyXgYPAN4jyUeTfD7JZUl+e0w/iiRpkVo67gIkSdpF3DvJ54a27w+c17w+GfiHqjo9yfOAtwFPmae/Q4GHVtVXkzwNuL6qngSQZNmIa5ckaU6O+EmSNHB7VT18+gt4zdC+RwHvbV6/G/jVFv19pqq+2ry+FHh8kjcl+bWq2jK6siVJmp/BT5Kkhavm+1aaf0uTBNh96D3f/dGbq74E/CKDAPjGJMOhUpKkzhn8JEma378DRzevnw38W/P6GgaBDuBIYLdtHZxkX+B7VXUG8GbgFzqrVJKkbfAeP0mS5vcS4NQkfwTcBBzTtL8L+HCSzwCfYGiUb4aHAX+Z5C7gh8AfdFyvJEl3k6qa/12SJEmSpInlVE9JkiRJ6jmDnyRJkiT1nMFPkiRJknrO4CdJkiRJPWfwkyRJkqSeM/hJkiRJUs8Z/CRJkiSp5wx+kiRJktRz/x/5vE2BD9hScAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1080x360 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plot_td_hist(deltas, bins=np.arange(0, 250, 1))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python",
"language": "python",
"name": ""
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment