Created
February 18, 2020 22:09
-
-
Save alimanfoo/66d7760440eba981214d483f6d917c79 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"from collections.abc import MutableMapping\n", | |
"\n", | |
"\n", | |
"class SafeStoreWrapper(MutableMapping):\n", | |
" \n", | |
" def __init__(self, store):\n", | |
" self.store = store\n", | |
" \n", | |
" def __getitem__(self, key):\n", | |
" try:\n", | |
" return self.store[key]\n", | |
" except KeyError as e:\n", | |
" # always raise a runtime error to ensure zarr propagates the exception\n", | |
" raise RuntimeError(e)\n", | |
" \n", | |
" def __setitem__(self, key, value):\n", | |
" self.store[key] = value\n", | |
" \n", | |
" def __delitem__(self, key):\n", | |
" del self.store[key]\n", | |
" \n", | |
" def __contains__(self, key):\n", | |
" return key in self.store\n", | |
" \n", | |
" def __iter__(self):\n", | |
" return iter(self.store)\n", | |
" \n", | |
" def __len__(self):\n", | |
" return len(self.store)\n", | |
" \n", | |
" " | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"# SafeStoreWrapper can be used to wrap any kind of zarr store, e.g., a GCSMap ...\n", | |
"\n", | |
"import gcsfs\n", | |
"gcs_bucket_fs = gcsfs.GCSFileSystem(project='malariagen-jupyterhub', token='anon', access='read_only')\n", | |
"storage_path = 'ag1000g-release/phase1.AR3/variation/main/zarr/ag1000g.phase1.ar3'\n", | |
"store = gcsfs.mapping.GCSMap(storage_path, gcs=gcs_bucket_fs, check=False, create=False)\n", | |
"safe_store = SafeStoreWrapper(store)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"<class 'KeyError'>\n" | |
] | |
} | |
], | |
"source": [ | |
"try:\n", | |
" # demonstrate what the original store does with a key that doesn't exist\n", | |
" store['foo']\n", | |
"except Exception as e:\n", | |
" print(type(e))" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"<class 'RuntimeError'>\n" | |
] | |
} | |
], | |
"source": [ | |
"try:\n", | |
" # demonstrate what the wrapped store does with a key that doesn't exist\n", | |
" safe_store['foo']\n", | |
"except Exception as e:\n", | |
" print(type(e))" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([ 103, 163, 192, ..., 49361791, 49361815, 49362329],\n", | |
" dtype=int32)" | |
] | |
}, | |
"execution_count": 5, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"# Use like this, can guarantee that zarr will raise exception if any chunk is missing, \n", | |
"# rather than fill in with fill value ...\n", | |
"import zarr\n", | |
"root = zarr.Group(store=safe_store)\n", | |
"pos = root['2L/variants/POS'][:]\n", | |
"pos" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"# etc. ..." | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.7.6" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 4 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment