get_all_invlists.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import faiss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make some ivf index\n",
    "index = faiss.index_factory(10, \"IVF10,Flat\")\n",
    "index.train(np.random.rand(1000, 10).astype('float32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add a few ids to it\n",
    "ids = np.array([10, 34, 100, 10, 225])\n",
    "x = np.random.rand(len(ids), 10).astype('float32')\n",
    "index.add_with_ids(x, ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_invlists(index): \n",
    "    index = faiss.extract_index_ivf(index)\n",
    "    invlists = index.invlists\n",
    "    all_ids = []\n",
    "    for listno in range(index.nlist): \n",
    "        ls = invlists.list_size(listno)\n",
    "        if ls == 0: continue\n",
    "        all_ids.append(\n",
    "            faiss.rev_swig_ptr(invlists.get_ids(listno), ls).copy()\n",
    "        )\n",
    "    return all_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([10]), array([ 34, 225]), array([100,  10])]"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "bento_obj_id": "139826113790984"
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get all the non-empty inverted lists\n",
    "get_invlists(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 10,  34, 225, 100,  10])"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "bento_obj_id": "139826105033872"
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# as a single list\n",
    "np.hstack(get_invlists(index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "bento_stylesheets": {
   "bento/extensions/flow/main.css": true,
   "bento/extensions/kernel_selector/main.css": true,
   "bento/extensions/kernel_ui/main.css": true,
   "bento/extensions/new_kernel/main.css": true,
   "bento/extensions/system_usage/main.css": true,
   "bento/extensions/theme/main.css": true
  },
  "kernelspec": {
   "display_name": "faiss",
   "language": "python",
   "name": "bento_kernel_faiss"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3rc1+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

Note that get_invlist is now supported via faiss.contrib.inspect_tools

https://github.com/facebookresearch/faiss/blob/master/contrib/inspect_tools.py#L9