Last active
April 2, 2017 01:51
-
-
Save rvernica/a5f8cd60e7231706c8cf324fed1116dc to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [], | |
"source": [ | |
"import scidbpy\n", | |
"import numpy\n", | |
"\n", | |
"db = scidbpy.connect()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [], | |
"source": [ | |
"db.query(\"\"\"\n", | |
" store(\n", | |
" build(\n", | |
" <x:int64, y:float>[i=1:2; j=2:3],\n", | |
"\n", | |
" '[[(1 , .1) , (?2, .2 )],\n", | |
" [(10, ?10), (20, .02)]]', true), foo)\"\"\")\n", | |
"\n", | |
"# use 'apply' instead of 'unpack'\n", | |
"db.query('store(apply(foo, dim_i, i, dim_j, j), bar)')" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Binary Download" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"'\\xff\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xff...'" | |
] | |
}, | |
"execution_count": 4, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"barray = db._scan_array('bar', \n", | |
" fmt = '(int64 NULL, float NULL, int64, int64)') \n", | |
"barray[:10] + '...'" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Map to NumPy Array" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([((255, 1), (255, 0.10000000149011612), 1, 2),\n", | |
" ((2, 0), (255, 0.20000000298023224), 1, 3),\n", | |
" ((255, 10), (10, 0.0), 2, 2),\n", | |
" ((255, 20), (255, 0.019999999552965164), 2, 3)], \n", | |
" dtype=[('x', [('null', 'u1'), ('val', '<i8')]), ('y', [('null', 'u1'), ('val', '<f4')]), ('i', '<i8'), ('j', '<i8')])" | |
] | |
}, | |
"execution_count": 5, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"foo = numpy.frombuffer(\n", | |
" barray,\n", | |
" [('x', [('null', 'uint8'), ('val', 'int64' )]), \n", | |
" ('y', [('null', 'uint8'), ('val', 'float32')]),\n", | |
" ('i', 'int64'),\n", | |
" ('j', 'int64')])\n", | |
"foo" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Map to Masked NumPy Array" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"masked_array(data = [(1, 0.10000000149011612, 1, 2) (--, 0.20000000298023224, 1, 3)\n", | |
" (10, --, 2, 2) (20, 0.019999999552965164, 2, 3)],\n", | |
" mask = [(False, False, False, False) (True, False, False, False)\n", | |
" (False, True, False, False) (False, False, False, False)],\n", | |
" fill_value = (999999, 1.0000000200408773e+20, 999999, 999999),\n", | |
" dtype = [('x', '<i8'), ('y', '<f4'), ('i', '<i8'), ('j', '<i8')])" | |
] | |
}, | |
"execution_count": 6, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"dtype = ([(t[0], t[1][1][1]) for t in foo.dtype.descr[:2]] + \n", | |
" foo.dtype.descr[2:])\n", | |
"\n", | |
"## copy mask\n", | |
"mask = numpy.full(foo.shape, \n", | |
" [False] * foo.size, \n", | |
" dtype = [(t[0], 'bool') for t in dtype])\n", | |
"mask['x'] = foo['x']['null'] != 255\n", | |
"mask['y'] = foo['y']['null'] != 255\n", | |
"\n", | |
"## copy data\n", | |
"data = numpy.empty(len(foo), dtype = dtype)\n", | |
"data['i'] = foo['i']\n", | |
"data['j'] = foo['j']\n", | |
"\n", | |
"null_x = numpy.where(mask['x'])\n", | |
"data['x'] = foo['x']['val']\n", | |
"data['x'][null_x] = foo['x']['null'][null_x] ## copy missing codes\n", | |
"\n", | |
"null_y = numpy.where(mask['y'])\n", | |
"data['y'] = foo['y']['val']\n", | |
"data['y'][null_y] = foo['y']['null'][null_y] ## copy missing codes\n", | |
"\n", | |
"## define masked array\n", | |
"foo_ma = numpy.ma.array(data = data, mask = mask, dtype = dtype)\n", | |
"foo_ma" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([(1, 0.10000000149011612, 1, 2), (2, 0.20000000298023224, 1, 3),\n", | |
" (10, 10.0, 2, 2), (20, 0.019999999552965164, 2, 3)], \n", | |
" dtype=[('x', '<i8'), ('y', '<f4'), ('i', '<i8'), ('j', '<i8')])" | |
] | |
}, | |
"execution_count": 7, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"foo_ma.data ## missing codes arailable" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### 2x2 Matrix" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": { | |
"collapsed": true, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [], | |
"source": [ | |
"db.query('remove(foo)')\n", | |
"db.query('remove(bar)')\n", | |
"db.query(\"\"\"\n", | |
" store(\n", | |
" build(\n", | |
" <x:float>[i=0:2; j=0:2],\n", | |
"\n", | |
" '[[(.1), (.2), (.3)],\n", | |
" [(.4), (.5), (.6)],\n", | |
" [(.7), (.8), (.9)]]', true), foo)\"\"\")\n", | |
"\n", | |
"# use 'apply' instead of 'unpack'\n", | |
"db.query('store(apply(foo, dim_i, i, dim_j, j), bar)')" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Binary Download" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"'\\xff\\xcd\\xcc\\xcc=\\x00\\x00\\x00\\x00\\x00...'" | |
] | |
}, | |
"execution_count": 9, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"barray = db._scan_array('bar', fmt = '(float NULL, int64, int64)')\n", | |
"\n", | |
"barray[:10] + '...'" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Map to NumPy Array and Reshape" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 10, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([((-1, 0.10000000149011612), 0, 0),\n", | |
" ((-1, 0.20000000298023224), 0, 1),\n", | |
" ((-1, 0.30000001192092896), 0, 2), ((-1, 0.4000000059604645), 1, 0),\n", | |
" ((-1, 0.5), 1, 1), ((-1, 0.6000000238418579), 1, 2),\n", | |
" ((-1, 0.699999988079071), 2, 0), ((-1, 0.800000011920929), 2, 1),\n", | |
" ((-1, 0.8999999761581421), 2, 2)], \n", | |
" dtype=[('x', [('null', 'i1'), ('val', '<f4')]), ('i', '<i8'), ('j', '<i8')])" | |
] | |
}, | |
"execution_count": 10, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"foo = numpy.frombuffer(\n", | |
" barray,\n", | |
" [('x', [('null', 'int8'), ('val', 'float32')]),\n", | |
" ('i', 'int64'),\n", | |
" ('j', 'int64')])\n", | |
"foo" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 11, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([[ 0.1 , 0.2 , 0.30000001],\n", | |
" [ 0.40000001, 0.5 , 0.60000002],\n", | |
" [ 0.69999999, 0.80000001, 0.89999998]], dtype=float32)" | |
] | |
}, | |
"execution_count": 11, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"foo['x']['val'].reshape((3,3))" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Binary Download Values-only" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 12, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"'\\xcd\\xcc\\xcc=\\xcd\\xccL>\\x9a\\x99...'" | |
] | |
}, | |
"execution_count": 12, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"barray = db._scan_array('foo', fmt = '(float)')\n", | |
"\n", | |
"barray[:10] + '...'" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": { | |
"deletable": true, | |
"editable": true | |
}, | |
"source": [ | |
"### Map to NumPy Array and Reshape" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 13, | |
"metadata": { | |
"collapsed": false, | |
"deletable": true, | |
"editable": true | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/plain": [ | |
"array([[ 0.1 , 0.2 , 0.30000001],\n", | |
" [ 0.40000001, 0.5 , 0.60000002],\n", | |
" [ 0.69999999, 0.80000001, 0.89999998]], dtype=float32)" | |
] | |
}, | |
"execution_count": 13, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"numpy.frombuffer(barray, 'float32').reshape((3,3))" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 2", | |
"language": "python", | |
"name": "python2" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 2 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython2", | |
"version": "2.7.13" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 2 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment