nwb conversion

generate_md.py

from jinja2 import Template

_str = """## Converting data to NWB
Below is a collection of simple conversion scripts that are all tested against small 
proprietary examples files. They are all optimized to handle very large data by iteratively
steping through large files and read/writing them one piece at a time. They also leverage
lossless compression within HDF5, which allows you to make large datasets smaller without
losing any data. We have seen this reduce large datasets by up to 66%!

{% for modality_name, modality_value in _dict.items() %}
<details>
<summary>{{modality_name}}</summary>
{{modality_value.msg}}
<blockquote>
<p>
    {% for type_name, type_dict in modality_value['data'].items() %}
<details>
<summary>{{type_name}}</summary><blockquote>
<p>
        {% for format_name, format_dict in type_dict.data.items() %}

<details>
<summary>    {{format_name}}</summary><blockquote>
<p>

```python
from {{modality_value.repo}} import Nwb{{type_name}}Extractor, {{format_dict.extractor}}

{{type_dict.object_name}} = {{format_dict.extractor}}("{{format_dict.dataset_path_arg}}")
{{type_dict.nwb_converter_class_name}}.{{type_dict.method_name}}({{type_dict.object_name}}, "output_path.nwb")
```
</p>
</blockquote></details>
        {% endfor %}
</p>
</blockquote></details>
    {% endfor %}
</p>
</blockquote></details>
{% endfor %}    
"""

_dict = {
    'Extracellular electrophysiology': dict(
        repo='spikeextractors',
        msg="""
<br>
For extracellular electrophysiology, we use the SpikeExtractors repository from the 
[SpikeInterface](http://spikeinterface.readthedocs.io/) 
project. To install this package, run

```bash
$ pip install spikeextractors
```
        
All of the format listed below are tested against example dataset in the 
[ephy_testing_data](https://gin.g-node.org/NeuralEnsemble/ephy_testing_data) GIN repository 
maintained by the NEO team.""",
        data=dict(
            Recording=dict(
                nwb_converter_class_name="NwbRecordingExtractor",
                method_name='write_recording',
                object_name='rx',
                data={
                    'Blackrock': dict(
                        extractor="BlackrockRecordingExtractor",
                        dataset_path_arg="dataset_path"
                    ),
                    'Intan': dict(
                        extractor="IntanRecordingExtractor",
                        dataset_path_arg="intan_rhd_test_1.rhd"
                    ),
                    'MEArec': dict(
                        extractor="MEArecRecordingExtractor",
                        dataset_path_arg="mearec_test_10s.h5"
                    ),
                    'Neuralynx': dict(
                        extractor="NeuralynxRecordingExtractor",
                        dataset_path_arg="data_directory"
                    ),
                    'Neuroscope': dict(
                        extractor="NeuroscopeRecordingExtractor",
                        dataset_path_arg="data_file.dat"
                    ),
                    'OpenEphys (legacy)': dict(
                        extractor="OpenEphysRecordingExtractor",
                        dataset_path_arg="data_folder"
                    ),
                    'OpenEphys binary (Neuropixels)': dict(
                        extractor="OpenEphysNPIXRecordingExtractor",
                        dataset_path_arg="folder_path"
                    ),
                    'Phy': dict(
                        extractor="PhyRecordingExtractor",
                        dataset_path_arg="folder_path"
                    ),
                    'SpikeGLX': dict(
                        extractor="SpikeGLXRecordingExtractor",
                        dataset_path_arg="MySession_g0_t0.imec0.ap.bin"
                    )
                }
            ),
            Sorting=dict(
                nwb_converter_class_name="NwbSortingExtractor",
                method_name='write_sorting',
                object_name='rx',
                data={
                    'Blackrock': dict(
                        extractor="BlackrockSortingExtractor",
                        dataset_path_arg="filename"
                    ),
                    "Klusta": dict(
                        extractor="KlustaSortingExtractor",
                        dataset_path_arg="neo.kwik"
                    ),
                    "MEArec": dict(
                        extractor="MEArecSortingExtractor",
                        dataset_path_arg="mearec_test_10s.h5"
                    ),
                    "Phy": dict(
                        extractor="PhySortingExtractor",
                        dataset_path_arg="data_folder"
                    ),
                    "Plexon": dict(
                        extarctor="PlexonSortingExtractor",
                        dataset_path_arg="File_plexon_2.plx"
                    ),
                    "Spyking Circus": dict(
                        extarctor="SpykingCircusSortingExtractor",
                        dataset_path_arg="file_or_folder_path"
                    )
                }
            )
        )
    ),
    'Optical physiology': dict(
        repo='roiextractors',
        msg="""
<br>
For optical physiology, we use the [RoiExtractors](https://roiextractors.readthedocs.io/en/latest/)
library developed by [CatalystNeuro](catalystneuro.com). To install, run

```bash
$ pip install roiextractors
``` 

All formats listed in the optical physiology section are tested against the 
[ophys_testing_data](https://gin.g-node.org/CatalystNeuro/ophys_testing_data) GIN repository.""",
        data=dict(
            Imaging=dict(
                nwb_converter_class_name='NwbImagingExtractor',
                method_name="write_imaging",
                object_name="imaging_ex",
                data=dict(
                    Tiff=dict(
                        extractor='TiffImagingExtractor',
                        dataset_path_arg='imaging.tiff'
                    ),
                    Hdf5=dict(
                        extractor='Hdf5ImagingExtractor',
                        dataset_path_arg='Movie.hdf5',
                    ),
                    SBX=dict(
                        extractor='SbxImagingExtractor',
                        dataset_path_arg='scanbox_file.mat'
                    )
                )
            ),
            Segmentation=dict(
                nwb_converter_class_name='NwbSegmentationExtractor',
                method_name="write_segmentation",
                object_name='seg_ex',
                data=dict(
                    CaImAn=dict(
                        extractor='CaimanSegmentationExtractor',
                        dataset_path_arg='caiman_analysis.hdf5'
                    ),
                    Suite2p=dict(
                        extractor='Suite2pSegmentationExtractor',
                        dataset_path_arg='segmentation_datasets/suite2p'
                    )
                )
            )
        )
    )
}

template = Template(_str)
print(template.render(_dict=_dict))

nwb_conversion.md

Converting data to NWB

Below is a collection of simple conversion scripts that are all tested against small proprietary examples files. They are all optimized to handle very large data by iteratively steping through large files and read/writing them one piece at a time. They also leverage lossless compression within HDF5, which allows you to make large datasets smaller without losing any data. We have seen this reduce large datasets by up to 66%!

Extracellular electrophysiology

For extracellular electrophysiology, we use the SpikeExtractors repository from the [SpikeInterface](http://spikeinterface.readthedocs.io/) project. To install this package, run

$ pip install spikeextractors

All of the format listed below are tested against example dataset in the ephy_testing_data GIN repository maintained by the NEO team.

Recording

Blackrock

from spikeextractors import NwbRecordingExtractor, BlackrockRecordingExtractor

rx = BlackrockRecordingExtractor("dataset_path")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

Intan

from spikeextractors import NwbRecordingExtractor, IntanRecordingExtractor

rx = IntanRecordingExtractor("intan_rhd_test_1.rhd")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

MEArec

from spikeextractors import NwbRecordingExtractor, MEArecRecordingExtractor

rx = MEArecRecordingExtractor("mearec_test_10s.h5")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

Neuralynx

from spikeextractors import NwbRecordingExtractor, NeuralynxRecordingExtractor

rx = NeuralynxRecordingExtractor("data_directory")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

Neuroscope

from spikeextractors import NwbRecordingExtractor, NeuroscopeRecordingExtractor

rx = NeuroscopeRecordingExtractor("data_file.dat")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

OpenEphys (legacy)

from spikeextractors import NwbRecordingExtractor, OpenEphysRecordingExtractor

rx = OpenEphysRecordingExtractor("data_folder")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

OpenEphys binary (Neuropixels)

from spikeextractors import NwbRecordingExtractor, OpenEphysNPIXRecordingExtractor

rx = OpenEphysNPIXRecordingExtractor("folder_path")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

Phy

from spikeextractors import NwbRecordingExtractor, PhyRecordingExtractor

rx = PhyRecordingExtractor("folder_path")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

SpikeGLX

from spikeextractors import NwbRecordingExtractor, SpikeGLXRecordingExtractor

rx = SpikeGLXRecordingExtractor("MySession_g0_t0.imec0.ap.bin")
NwbRecordingExtractor.write_recording(rx, "output_path.nwb")

Sorting

Blackrock

from spikeextractors import NwbSortingExtractor, BlackrockSortingExtractor

rx = BlackrockSortingExtractor("filename")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

Klusta

from spikeextractors import NwbSortingExtractor, KlustaSortingExtractor

rx = KlustaSortingExtractor("neo.kwik")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

MEArec

from spikeextractors import NwbSortingExtractor, MEArecSortingExtractor

rx = MEArecSortingExtractor("mearec_test_10s.h5")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

Phy

from spikeextractors import NwbSortingExtractor, PhySortingExtractor

rx = PhySortingExtractor("data_folder")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

Plexon

from spikeextractors import NwbSortingExtractor, 

rx = ("File_plexon_2.plx")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

Spyking Circus

from spikeextractors import NwbSortingExtractor, 

rx = ("file_or_folder_path")
NwbSortingExtractor.write_sorting(rx, "output_path.nwb")

Optical physiology

For optical physiology, we use the [RoiExtractors](https://roiextractors.readthedocs.io/en/latest/) library developed by [CatalystNeuro](catalystneuro.com). To install, run

$ pip install roiextractors

All formats listed in the optical physiology section are tested against the ophys_testing_data GIN repository.

Imaging

Tiff

from roiextractors import NwbImagingExtractor, TiffImagingExtractor

imaging_ex = TiffImagingExtractor("imaging.tiff")
NwbImagingExtractor.write_imaging(imaging_ex, "output_path.nwb")

Hdf5

from roiextractors import NwbImagingExtractor, Hdf5ImagingExtractor

imaging_ex = Hdf5ImagingExtractor("Movie.hdf5")
NwbImagingExtractor.write_imaging(imaging_ex, "output_path.nwb")

SBX

from roiextractors import NwbImagingExtractor, SbxImagingExtractor

imaging_ex = SbxImagingExtractor("scanbox_file.mat")
NwbImagingExtractor.write_imaging(imaging_ex, "output_path.nwb")

Segmentation

CaImAn

from roiextractors import NwbSegmentationExtractor, CaimanSegmentationExtractor

seg_ex = CaimanSegmentationExtractor("caiman_analysis.hdf5")
NwbSegmentationExtractor.write_segmentation(seg_ex, "output_path.nwb")

Suite2p

from roiextractors import NwbSegmentationExtractor, Suite2pSegmentationExtractor

seg_ex = Suite2pSegmentationExtractor("segmentation_datasets/suite2p")
NwbSegmentationExtractor.write_segmentation(seg_ex, "output_path.nwb")

That is awesome

nice work! we should add this info somewhere in the dandi handbook on converting to NWB.