Process SpikeForest datasets

driver_process_datasets.py

import argparse
import spikeforest as sf
import os

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Process the SpikeForest datasets')
    parser.add_argument('command',help='download, process, clear_in_process, clear_all, assemble')
    args = parser.parse_args()

    spikeforest_password=os.environ.get('SPIKEFOREST_PASSWORD','')
    if not spikeforest_password:
      raise Exception('Environment variable not set: SPIKEFOREST_PASSWORD')
    
    sf.kbucketConfigRemote(
      share_id='spikeforest.spikeforest1',
      write=True,
      password=spikeforest_password
    )

    command=args.command
    if command=='clear_in_process':
      sf.batch.clear_dataset_results(in_process_only=True)
    elif command=='clear_all':
      sf.batch.clear_dataset_results(in_process_only=False)
    elif command=='download':
      sf.batch.download_datasets()
    elif command=='process':
      sf.batch.process_datasets()
    elif command=='assemble':
      sf.batch.assemble_dataset_results()
    else:
      raise Exception('Unrecognized command: '+command)

notebook.ipynb

run.sh

#!/bin/bash

set -e

python driver_process_datasets.py download
python driver_process_datasets.py clear_in_process
srun -c2 -n 20 python driver_process_datasets.py process
python driver_process_datasets.py assemble