Skip to content

Instantly share code, notes, and snippets.

@scottyhq

scottyhq/s3fs-vs-gcsfs.ipynb

Created February 27, 2019 01:03
Show Gist options
  • Save scottyhq/304a3c4b4e198776b8d82fb3a9f300e3 to your computer and use it in GitHub Desktop.
Save scottyhq/304a3c4b4e198776b8d82fb3a9f300e3 to your computer and use it in GitHub Desktop.
Download ZIP
testing netcdf4 access with s3fs versus gcsfs
Display the source blob
Display the rendered blob
Raw
s3fs-vs-gcsfs.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# GCSFS"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"_call out of retries on exception: HTTPConnectionPool(host='metadata.google.internal', port=80): Max retries exceeded with url: /computeMetadata/v1/instance/service-accounts/default/?recursive=true (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known',))\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connection.py\", line 159, in _new_conn\n",
" (self._dns_host, self.port), self.timeout, **extra_kw)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/util/connection.py\", line 57, in create_connection\n",
" for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/socket.py\", line 745, in getaddrinfo\n",
" for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\n",
"socket.gaierror: [Errno 8] nodename nor servname provided, or not known\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\n",
" chunked=chunked)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\n",
" conn.request(method, url, **httplib_request_kw)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/http/client.py\", line 1239, in request\n",
" self._send_request(method, url, body, headers, encode_chunked)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/http/client.py\", line 1285, in _send_request\n",
" self.endheaders(body, encode_chunked=encode_chunked)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/http/client.py\", line 1234, in endheaders\n",
" self._send_output(message_body, encode_chunked=encode_chunked)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/http/client.py\", line 1026, in _send_output\n",
" self.send(msg)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/http/client.py\", line 964, in send\n",
" self.connect()\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connection.py\", line 181, in connect\n",
" conn = self._new_conn()\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connection.py\", line 168, in _new_conn\n",
" self, \"Failed to establish a new connection: %s\" % e)\n",
"urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/requests/adapters.py\", line 449, in send\n",
" timeout=timeout\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\n",
" _stacktrace=sys.exc_info()[2])\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/urllib3/util/retry.py\", line 398, in increment\n",
" raise MaxRetryError(_pool, url, error or ResponseError(cause))\n",
"urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='metadata.google.internal', port=80): Max retries exceeded with url: /computeMetadata/v1/instance/service-accounts/default/?recursive=true (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known',))\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/transport/requests.py\", line 120, in __call__\n",
" **kwargs)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/requests/sessions.py\", line 533, in request\n",
" resp = self.send(prep, **send_kwargs)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/requests/sessions.py\", line 646, in send\n",
" r = adapter.send(request, **kwargs)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/requests/adapters.py\", line 516, in send\n",
" raise ConnectionError(e, request=request)\n",
"requests.exceptions.ConnectionError: HTTPConnectionPool(host='metadata.google.internal', port=80): Max retries exceeded with url: /computeMetadata/v1/instance/service-accounts/default/?recursive=true (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known',))\n",
"\n",
"The above exception was the direct cause of the following exception:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/compute_engine/credentials.py\", line 96, in refresh\n",
" self._retrieve_info(request)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/compute_engine/credentials.py\", line 78, in _retrieve_info\n",
" service_account=self._service_account_email)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/compute_engine/_metadata.py\", line 186, in get_service_account_info\n",
" recursive=True)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/compute_engine/_metadata.py\", line 122, in get\n",
" response = request(url=url, method='GET', headers=_METADATA_HEADERS)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/transport/requests.py\", line 124, in __call__\n",
" six.raise_from(new_exc, caught_exc)\n",
" File \"<string>\", line 3, in raise_from\n",
"google.auth.exceptions.TransportError: HTTPConnectionPool(host='metadata.google.internal', port=80): Max retries exceeded with url: /computeMetadata/v1/instance/service-accounts/default/?recursive=true (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known',))\n",
"\n",
"The above exception was the direct cause of the following exception:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/gcsfs/core.py\", line 471, in _call\n",
" r = meth(self.base + path, params=kwargs, json=json)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/requests/sessions.py\", line 546, in get\n",
" return self.request('GET', url, **kwargs)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/transport/requests.py\", line 205, in request\n",
" self._auth_request, method, url, request_headers)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/credentials.py\", line 122, in before_request\n",
" self.refresh(request)\n",
" File \"/Users/scott/miniconda3/envs/test_env/lib/python3.6/site-packages/google/auth/compute_engine/credentials.py\", line 102, in refresh\n",
" six.raise_from(new_exc, caught_exc)\n",
" File \"<string>\", line 3, in raise_from\n",
"google.auth.exceptions.RefreshError: HTTPConnectionPool(host='metadata.google.internal', port=80): Max retries exceeded with url: /computeMetadata/v1/instance/service-accounts/default/?recursive=true (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11d9170f0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known',))\n"
]
}
],
"source": [
"#throws lots of warnings first time run\n",
"import gcsfs\n",
"import xarray as xr\n",
"\n",
"fs = gcsfs.GCSFileSystem() #default anonymous access\n",
"filename = 'S1-GUNW-A-R-137-tops-20181129_20181123-020010-43220N_41518N-PP-e2c7-v2_0_0.nc'\n",
"fileObj = fs.open(f'pangeo-data/grfn-v2/137/{filename}') "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (latitude: 2045, longitude: 4158)\n",
"Coordinates:\n",
" * longitude (longitude) float64 -123.1 -123.1 ... -119.6 -119.6\n",
" * latitude (latitude) float64 43.22 43.22 43.22 ... 41.52 41.52\n",
"Data variables:\n",
" crs int32 ...\n",
" unwrappedPhase (latitude, longitude) float32 ...\n",
" coherence (latitude, longitude) float32 ...\n",
" connectedComponents (latitude, longitude) float32 ...\n",
" amplitude (latitude, longitude) float32 ..."
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds = xr.open_dataset(fileObj, group='/science/grids/data', engine='h5netcdf')\n",
"ds"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# S3FS"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import s3fs\n",
"import json\n",
"import xarray as xr\n",
"\n",
"# get temporary credentials for s3 access\n",
"credential_url = 'https://grfn.asf.alaska.edu/door/credentials'\n",
"response = requests.get(credential_url)\n",
"response.raise_for_status()\n",
"credentials = json.loads(response.text)['Credentials']\n",
"\n",
"fs = s3fs.S3FileSystem(key=credentials['AccessKeyId'], \n",
" secret=credentials['SecretAccessKey'], \n",
" token=credentials['SessionToken'])\n",
"\n",
"s3Path = f'grfn-content-prod/{filename}'\n",
"\n",
"fileObj = fs.open(s3Path)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"ename": "SystemError",
"evalue": "PyEval_EvalFrameEx returned a result with an error set",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/file_manager.py\u001b[0m in \u001b[0;36macquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 166\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 167\u001b[0;31m \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_key\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 168\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/lru_cache.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 42\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_to_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyError\u001b[0m: [<function _open_h5netcdf_group at 0x11d8b0ae8>, (<S3File grfn-content-prod/S1-GUNW-A-R-137-tops-20181129_20181123-020010-43220N_41518N-PP-e2c7-v2_0_0.nc>,), 'r', (('group', '/science/grids/data'),)]",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 1498\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1499\u001b[0;31m \u001b[0mb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1500\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/h5fd.cpython-36m-darwin.so\u001b[0m in \u001b[0;36mView.MemoryView.memoryview.__setitem__\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/h5fd.cpython-36m-darwin.so\u001b[0m in \u001b[0;36mView.MemoryView.memoryview.setitem_slice_assignment\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/h5fd.cpython-36m-darwin.so\u001b[0m in \u001b[0;36mView.MemoryView.memoryview_copy_contents\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/h5fd.cpython-36m-darwin.so\u001b[0m in \u001b[0;36mView.MemoryView._err_extents\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: got differing extents in dimension 0 (got 8 and 59941567)",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mSystemError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-0d5ca329530c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfileObj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'/science/grids/data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'h5netcdf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/api.py\u001b[0m in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime)\u001b[0m\n\u001b[1;32m 362\u001b[0m raise ValueError(\"byte header doesn't match netCDF3 or \"\n\u001b[1;32m 363\u001b[0m \"netCDF4/HDF5: {}\".format(magic_number))\n\u001b[0;32m--> 364\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmaybe_decode_store\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 366\u001b[0m \u001b[0;31m# Ensure source filename always stored in dataset object (GH issue #2550)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/api.py\u001b[0m in \u001b[0;36mmaybe_decode_store\u001b[0;34m(store, lock)\u001b[0m\n\u001b[1;32m 280\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask_and_scale\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmask_and_scale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecode_times\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecode_times\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 281\u001b[0m \u001b[0mconcat_characters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconcat_characters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecode_coords\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecode_coords\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 282\u001b[0;31m drop_variables=drop_variables, use_cftime=use_cftime)\n\u001b[0m\u001b[1;32m 283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0m_protect_dataset_variables_inplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/conventions.py\u001b[0m in \u001b[0;36mdecode_cf\u001b[0;34m(obj, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime)\u001b[0m\n\u001b[1;32m 468\u001b[0m \u001b[0mencoding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 469\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAbstractDataStore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 470\u001b[0;31m \u001b[0mvars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 471\u001b[0m \u001b[0mextra_coords\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 472\u001b[0m \u001b[0mfile_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/common.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 118\u001b[0m \"\"\"\n\u001b[1;32m 119\u001b[0m variables = FrozenOrderedDict((_decode_variable_name(k), v)\n\u001b[0;32m--> 120\u001b[0;31m for k, v in self.get_variables().items())\n\u001b[0m\u001b[1;32m 121\u001b[0m \u001b[0mattributes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFrozenOrderedDict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_attrs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattributes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/h5netcdf_.py\u001b[0m in \u001b[0;36mget_variables\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 136\u001b[0m return FrozenOrderedDict((k, self.open_store_variable(k, v))\n\u001b[0;32m--> 137\u001b[0;31m for k, v in self.ds.variables.items())\n\u001b[0m\u001b[1;32m 138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_attrs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/h5netcdf_.py\u001b[0m in \u001b[0;36mds\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mopen_store_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/file_manager.py\u001b[0m in \u001b[0;36macquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mode'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 173\u001b[0;31m \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 174\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0;31m# ensure file doesn't get overriden when opened again\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Documents/GitHub/xarray/xarray/backends/h5netcdf_.py\u001b[0m in \u001b[0;36m_open_h5netcdf_group\u001b[0;34m(filename, mode, group)\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_h5netcdf_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mh5netcdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5netcdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 63\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mclose_on_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 64\u001b[0m ds = _nc4_require_group(\n",
"\u001b[0;32m~/Documents/GitHub/h5netcdf/h5netcdf/core.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, mode, invalid_netcdf, **kwargs)\u001b[0m\n\u001b[1;32m 603\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 604\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_preexisting_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'a'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_h5file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5py\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 606\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 607\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_closed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, **kwds)\u001b[0m\n\u001b[1;32m 392\u001b[0m fid = make_fid(name, mode, userblock_size,\n\u001b[1;32m 393\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfcpl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmake_fcpl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrack_order\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack_order\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m swmr=swmr)\n\u001b[0m\u001b[1;32m 395\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 396\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mswmr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 169\u001b[0m \u001b[0mflags\u001b[0m \u001b[0;34m|=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_SWMR_READ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 170\u001b[0;31m \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 171\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_RDWR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mh5py/h5f.pyx\u001b[0m in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mh5py/defs.pyx\u001b[0m in \u001b[0;36mh5py.defs.H5Fopen\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32mh5py/h5fd.pyx\u001b[0m in \u001b[0;36mh5py.h5fd.H5FD_fileobj_read\u001b[0;34m()\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/test_env/lib/python3.6/site-packages/s3fs/core.py\u001b[0m in \u001b[0;36mseek\u001b[0;34m(self, loc, whence)\u001b[0m\n\u001b[1;32m 1234\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mstart\u001b[0m \u001b[0mof\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent\u001b[0m \u001b[0mlocation\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mend\u001b[0m \u001b[0mof\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1235\u001b[0m \"\"\"\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Seek only available in read mode'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1238\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mwhence\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mSystemError\u001b[0m: PyEval_EvalFrameEx returned a result with an error set"
]
}
],
"source": [
"ds = xr.open_dataset(fileObj, group='/science/grids/data', engine='h5netcdf')\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment