scidbpy AFL Prototype

SciDB_afl.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "SciDB AFL Binding Prototype"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Goal** Provide a 1-to-1 functional binding between the [AFL Operators](http://scidb.org/HTMLmanual/13.12/scidb_ug/ch17.html) and Python"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "from scidbpy import interface, SciDBQueryError, SciDBArray\n",
      "sdb = interface.SciDBShimInterface('http://192.168.56.101:8080')\n",
      "\n",
      "from scidbpy import afl"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "_X = np.random.random((10, 5))\n",
      "X = sdb.from_array(_X)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Basic Use"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q = afl.sum(X)\n",
      "q"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "Op<sum(py1100936456554_00001)>"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q.query  # Query string"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "'sum(py1100936456554_00001)'"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q.toarray()  # actually send query, evaluate, pack result into SciDBArray, convert to numpy"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "array([ 23.95540656])"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Operator Composition"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q = afl.sum(afl.apply(X, 'times2', 'f0*2'), 'times2')  # no database communication yet"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q.query"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "'sum(apply(py1100936456554_00001,times2,f0*2),times2)'"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "q.toarray()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "array([ 47.91081313])"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[o for o in afl.__dict__.values() if isinstance(o, type) and issubclass(o, afl.AFLOperator)]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "[scidbpy.afl.help,\n",
        " scidbpy.afl.show,\n",
        " scidbpy.afl.variable_window,\n",
        " scidbpy.afl.mpi_test,\n",
        " scidbpy.afl.window,\n",
        " scidbpy.afl.diskinfo,\n",
        " scidbpy.afl.attribute_rename,\n",
        " scidbpy.afl.save,\n",
        " scidbpy.afl.gemm,\n",
        " scidbpy.afl.bernoulli,\n",
        " scidbpy.afl.build_sparse,\n",
        " scidbpy.afl.index_lookup,\n",
        " scidbpy.afl.cumulate,\n",
        " scidbpy.afl.explain_logical,\n",
        " scidbpy.afl.materialize,\n",
        " scidbpy.afl.insert,\n",
        " scidbpy.afl.create_array,\n",
        " scidbpy.afl.allversions,\n",
        " scidbpy.afl.list,\n",
        " scidbpy.afl.mpi_init,\n",
        " scidbpy.afl.remove,\n",
        " scidbpy.afl.merge,\n",
        " scidbpy.afl.old_unpack,\n",
        " scidbpy.afl.instance_stats,\n",
        " scidbpy.afl.avg_rank,\n",
        " scidbpy.afl.repart,\n",
        " scidbpy.afl.substitute,\n",
        " scidbpy.afl.slice,\n",
        " scidbpy.afl.dimensions,\n",
        " scidbpy.afl.cast,\n",
        " scidbpy.afl.sum,\n",
        " scidbpy.afl.between,\n",
        " scidbpy.afl.subarray,\n",
        " scidbpy.afl.xgrid,\n",
        " scidbpy.afl.transpose,\n",
        " scidbpy.afl.deldim,\n",
        " scidbpy.afl.stdev,\n",
        " scidbpy.afl.explain_physical,\n",
        " scidbpy.afl.reverse,\n",
        " scidbpy.afl.filter,\n",
        " scidbpy.afl.attributes,\n",
        " scidbpy.afl.load,\n",
        " scidbpy.afl.consume,\n",
        " scidbpy.afl.rank,\n",
        " scidbpy.afl.echo,\n",
        " scidbpy.afl.cancel,\n",
        " scidbpy.afl.apply,\n",
        " scidbpy.afl.uniq,\n",
        " scidbpy.afl.redimension,\n",
        " scidbpy.afl.quantile,\n",
        " scidbpy.afl.var,\n",
        " scidbpy.afl.analyze,\n",
        " scidbpy.afl.store,\n",
        " scidbpy.afl.sort,\n",
        " scidbpy.afl.setopt,\n",
        " scidbpy.afl.load_library,\n",
        " scidbpy.afl.splitarraytest,\n",
        " scidbpy.afl.input,\n",
        " scidbpy.afl.cross_join,\n",
        " scidbpy.afl.unpack,\n",
        " scidbpy.afl.hello_instances,\n",
        " scidbpy.afl.count,\n",
        " scidbpy.afl.join,\n",
        " scidbpy.afl.versions,\n",
        " scidbpy.afl.project,\n",
        " scidbpy.afl.reduce_distro,\n",
        " scidbpy.afl.ApproxDC,\n",
        " scidbpy.afl.thin,\n",
        " scidbpy.afl.rename,\n",
        " scidbpy.afl.scan,\n",
        " scidbpy.afl.sample,\n",
        " scidbpy.afl.avg,\n",
        " scidbpy.afl.normalize,\n",
        " scidbpy.afl.min,\n",
        " scidbpy.afl.reshape,\n",
        " scidbpy.afl.cross,\n",
        " scidbpy.afl.regrid,\n",
        " scidbpy.afl.lookup,\n",
        " scidbpy.afl.build,\n",
        " scidbpy.afl.redimension_store,\n",
        " scidbpy.afl.max,\n",
        " scidbpy.afl.adddim,\n",
        " scidbpy.afl.AFLOperator,\n",
        " scidbpy.afl.aggregate,\n",
        " scidbpy.afl.unload_library,\n",
        " scidbpy.afl.gesvd,\n",
        " scidbpy.afl.concat,\n",
        " scidbpy.afl.mpicopy,\n",
        " scidbpy.afl.mpirank,\n",
        " scidbpy.afl.mstat,\n",
        " scidbpy.afl.sg]"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Todo:\n",
      "\n",
      " * Argument / type checking\n",
      " * Convenience functions to refer to attribute or dimension names\n",
      " * Query pre-processing or optimization?\n",
      " * Pull in operator documentation from SciDB source\n",
      " * Basic operations like +,-,%,..."
     ]
    }
   ],
   "metadata": {}
  }
 ]
}