Skip to content

Instantly share code, notes, and snippets.

@magland

magland/driver_sort_datasets.py

Last active November 6, 2018 18:15
Show Gist options
  • Save magland/3ba2b1fe6ff138deba0edaedb5de5867 to your computer and use it in GitHub Desktop.
Save magland/3ba2b1fe6ff138deba0edaedb5de5867 to your computer and use it in GitHub Desktop.
Download ZIP
3ba2b1fe6ff138deba0edaedb5de5867
Raw
driver_sort_datasets.py
import argparse
import spikeforest as sf
import os
if __name__ == '__main__':
parser = argparse.ArgumentParser(description = 'Sort the SpikeForest datasets')
parser.add_argument('command',help='sort, clear_in_process, clear_all, assemble')
parser.add_argument('study',help='The name of the study')
args = parser.parse_args()
spikeforest_password=os.environ.get('SPIKEFOREST_PASSWORD','')
if not spikeforest_password:
raise Exception('Environment variable not set: SPIKEFOREST_PASSWORD')
sf.kbucketConfigRemote(
share_id='spikeforest.spikeforest1',
write=True,
password=spikeforest_password
)
command=args.command
study_name=args.study
if command=='clear_in_process':
sf.batch.clear_sorting_results(study_name=study_name,in_process_only=True)
elif command=='clear_all':
sf.batch.clear_sorting_results(study_name=study_name,in_process_only=False)
elif command=='sort':
sf.batch.sort_datasets(study_name=study_name)
elif command=='assemble':
sf.batch.assemble_sorting_results(study_name=study_name)
else:
raise Exception('Unrecognized command: '+command)
Display the source blob
Display the rendered blob
Raw
notebook.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "3ba2b1fe6ff138deba0edaedb5de5867",
"version": "0.3.2",
"provenance": [],
"collapsed_sections": [],
"include_colab_link": true
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/magland/3ba2b1fe6ff138deba0edaedb5de5867/notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"metadata": {
"id": "WibF-QOm7CUk",
"colab_type": "text"
},
"cell_type": "markdown",
"source": [
"## Sort datasets\n",
"\n",
"[This gist](https://gist.github.com/magland/3ba2b1fe6ff138deba0edaedb5de5867) contains the scripts for running the SpikeForest spike sorting. This particular notebook demonstrates how to load the results for a particular study and show the accuracy plot."
]
},
{
"metadata": {
"id": "_eRW5RTg7aSl",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"## Do not run this cell unless you are on a hosted runtime that does not have these packages installed\n",
"%%capture\n",
"!pip install spikeforest\n",
"!pip install pandas"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "DUi99HkY6_wl",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"import spikeforest as sf\n",
"from kbucket import client as kb\n",
"from IPython.display import HTML\n",
"import pandas\n",
"import json"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "FEcpVJYl6_ww",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# A convenience function\n",
"def read_text_file(path):\n",
" with open(path,'r') as f:\n",
" return f.read()"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "FtBmOH706_wy",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# Configure the kbucket to look at the spikeforest1 server\n",
"sf.kbucketConfigRemote(share_id='spikeforest.spikeforest1')"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "-3LP5whN6_wz",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# Load the sorting results for a particular study\n",
"study_name='magland_synth_noise20_K10_C4'\n",
"obj=kb.loadObject(key=dict(name='spikeforest_sorting_results',study=study_name))\n",
"sorting_results=obj['sorting_results']"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "dw2tSLVe6_w6",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 247
},
"outputId": "e2bdd99f-7ecb-46c9-ab8a-924df841aa31"
},
"cell_type": "code",
"source": [
"# Grab and display a particular result\n",
"result0=sorting_results[2]['result']\n",
"display(result0)"
],
"execution_count": 9,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"{'comparison_with_truth': {'html': 'sha1://dee734d6506f913d3f04271661225acb17358cf1/table.html',\n",
" 'json': 'sha1://1e61182ef1cc85a2216bbffc8675f6c866e67f93/table.json'},\n",
" 'dataset_dir': 'kbucket://15734439d8cf/groundtruth/magland_synth/datasets_noise20_K10_C4/002_synth',\n",
" 'dataset_name': '002_synth',\n",
" 'firings': '/mnt/ceph/users/magland/sha1-cache/d/2b/d2b8451c7383100fdf3660a109fb6c22cfeae80a',\n",
" 'firings_true': 'kbucket://15734439d8cf/groundtruth/magland_synth/datasets_noise20_K10_C4/002_synth/firings_true.mda',\n",
" 'sorting_params': {'adjacency_radius': -1,\n",
" 'detect_sign': -1,\n",
" 'detect_threshold': 3},\n",
" 'sorting_processor_name': 'MountainSort4',\n",
" 'sorting_processor_version': '4.0.1',\n",
" 'summary': {'plots': {'autocorrelograms': 'sha1://4e7653439c517e3df252735f81c5be70b861ecdb/autocorrelograms.jpg',\n",
" 'unit_waveforms': 'sha1://aca9a032601a6aa1bdbe459567cbd686aa4df7c0/unit_waveforms.jpg'}}}"
]
},
"metadata": {
"tags": []
}
}
]
},
{
"metadata": {
"id": "j7Xm-H5Y6_w9",
"colab_type": "code",
"colab": {}
},
"cell_type": "code",
"source": [
"# Let's look at the comparison with ground truth\n",
"comparison_html=read_text_file(kb.realizeFile(result0['comparison_with_truth']['html']))\n",
"comparison_obj=json.loads(read_text_file(kb.realizeFile(result0['comparison_with_truth']['json'])))"
],
"execution_count": 0,
"outputs": []
},
{
"metadata": {
"id": "ciuIEy6B6_xA",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 347
},
"outputId": "7542834d-c8d8-4899-d444-2e8b895f170c"
},
"cell_type": "code",
"source": [
"#display(HTML(comparison_html))\n",
"df=pandas.DataFrame(data=comparison_obj).transpose()\n",
"display(df)"
],
"execution_count": 19,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th># matches</th>\n",
" <th>Accuracy</th>\n",
" <th>Best unit</th>\n",
" <th>Matched unit</th>\n",
" <th>Unit ID</th>\n",
" <th>f.n.</th>\n",
" <th>f.p.</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1386</td>\n",
" <td>0.91</td>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>0.06</td>\n",
" <td>0.03</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1289</td>\n",
" <td>0.89</td>\n",
" <td>3</td>\n",
" <td>3</td>\n",
" <td>2</td>\n",
" <td>0.04</td>\n",
" <td>0.08</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1359</td>\n",
" <td>0.97</td>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>3</td>\n",
" <td>0.00</td>\n",
" <td>0.03</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1382</td>\n",
" <td>0.97</td>\n",
" <td>4</td>\n",
" <td>4</td>\n",
" <td>4</td>\n",
" <td>0.01</td>\n",
" <td>0.02</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1378</td>\n",
" <td>0.93</td>\n",
" <td>5</td>\n",
" <td>5</td>\n",
" <td>5</td>\n",
" <td>0.03</td>\n",
" <td>0.04</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>872</td>\n",
" <td>0.62</td>\n",
" <td>13</td>\n",
" <td>13</td>\n",
" <td>6</td>\n",
" <td>0.00</td>\n",
" <td>0.38</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>1405</td>\n",
" <td>0.99</td>\n",
" <td>12</td>\n",
" <td>12</td>\n",
" <td>7</td>\n",
" <td>0.01</td>\n",
" <td>0.00</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>724</td>\n",
" <td>0.52</td>\n",
" <td>9</td>\n",
" <td>9</td>\n",
" <td>8</td>\n",
" <td>0.01</td>\n",
" <td>0.48</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>1387</td>\n",
" <td>0.98</td>\n",
" <td>11</td>\n",
" <td>11</td>\n",
" <td>9</td>\n",
" <td>0.00</td>\n",
" <td>0.02</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>1367</td>\n",
" <td>0.95</td>\n",
" <td>15</td>\n",
" <td>15</td>\n",
" <td>10</td>\n",
" <td>0.04</td>\n",
" <td>0.02</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" # matches Accuracy Best unit Matched unit Unit ID f.n. f.p.\n",
"0 1386 0.91 2 2 1 0.06 0.03\n",
"1 1289 0.89 3 3 2 0.04 0.08\n",
"2 1359 0.97 6 6 3 0.00 0.03\n",
"3 1382 0.97 4 4 4 0.01 0.02\n",
"4 1378 0.93 5 5 5 0.03 0.04\n",
"5 872 0.62 13 13 6 0.00 0.38\n",
"6 1405 0.99 12 12 7 0.01 0.00\n",
"7 724 0.52 9 9 8 0.01 0.48\n",
"8 1387 0.98 11 11 9 0.00 0.02\n",
"9 1367 0.95 15 15 10 0.04 0.02"
]
},
"metadata": {
"tags": []
}
}
]
}
]
}
Raw
run.sh
#!/bin/bash
set -e
# Set number of cpu's to use for spike sorting
export NUM_WORKERS=2
export MKL_NUM_THREADS=$NUM_WORKERS
export NUMEXPR_NUM_THREADS=$NUM_WORKERS
export OMP_NUM_THREADS=$NUM_WORKERS
#study=magland_synth_noise10_K10_C4
study=magland_synth_noise20_K10_C4
python driver_sort_datasets.py clear_in_process $study
srun -c 2 -n 40 python driver_sort_datasets.py sort $study
#srun -c 2 -n 1 python driver_sort_datasets.py sort $study
#python driver_sort_datasets.py sort $study
python driver_sort_datasets.py assemble $study
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment