d-v-b

from __future__ import annotations

from typing import (
    Any, 
    Dict, Generic, Iterable, Literal, TypeVar,
    TypedDict, Union, Protocol, runtime_checkable)
from pydantic import ValidationError
from pydantic.generics import GenericModel
from zarr.storage import init_group, BaseStore
import zarr

d-v-b

import zarr
import napari
import click

# Challenge napari with some big data. 
# Compare with how neuroglancer handles the same dataset: 
# http://neuroglancer-demo.appspot.com/#!%7B%22dimensions%22:%7B%22x%22:%5B4e-9%2C%22m%22%5D%2C%22y%22:%5B4e-9%2C%22m%22%5D%2C%22z%22:%5B4e-9%2C%22m%22%5D%7D%2C%22position%22:%5B5877.4033203125%2C7050.5%2C5252.34033203125%5D%2C%22crossSectionScale%22:36.598234443678%2C%22projectionScale%22:16384%2C%22layers%22:%5B%7B%22type%22:%22image%22%2C%22source%22:%22n5://https://janelia-cosem-datasets.s3.amazonaws.com/jrc_fly-fsb-1/jrc_fly-fsb-1.n5/em/fibsem-uint16%22%2C%22tab%22:%22rendering%22%2C%22shaderControls%22:%7B%22normalized%22:%7B%22range%22:%5B1773%2C4458%5D%7D%7D%2C%22name%22:%22fibsem-uint16%22%7D%5D%2C%22selectedLayer%22:%7B%22visible%22:true%2C%22layer%22:%22fibsem-uint16%22%7D%2C%22layout%22:%224panel%22%7D

containerPath = 's3://janelia-cosem-datasets/jrc_fly-fsb-1/jrc_fly-fsb-1.n5'
groupPath = 'em/fibsem-uint16'

d-v-b

[raw em data, pm predictions, mito predictions, er predictions](http://neuroglancer-demo.appspot.com/#!%7B%22dimensions%22:%7B%22x%22:%5B8e-9%2C%22m%22%5D%2C%22y%22:%5B8e-9%2C%22m%22%5D%2C%22z%22:%5B8e-9%2C%22m%22%5D%7D%2C%22position%22:%5B1470.5701904296875%2C664.328369140625%2C1278.6429443359375%5D%2C%22crossSectionScale%22:3.320116922736545%2C%22projectionScale%22:8192%2C%22layers%22:%5B%7B%22type%22:%22image%22%2C%22source%22:%22n5://https://janelia-aic-desmosome-2.s3.amazonaws.com/aic_desmosome-2.n5/volumes/raw%22%2C%22tab%22:%22source%22%2C%22name%22:%22raw%22%2C%22archived%22:true%7D%2C%7B%22type%22:%22image%22%2C%22source%22:%22n5://https://janelia-aic-desmosome-2.s3.amazonaws.com/aic_desmosome-2.n5/volumes/raw%22%2C%22tab%22:%22source%22%2C%22name%22:%22raw1%22%7D%2C%7B%22type%22:%22image%22%2C%22source%22:%22n5://https://janelia-aic-desmosome-2.s3.amazonaws.com/aic_desmosome-2.n5/volumes/labels/pm_pred%22%2C%22tab%22:%22rendering%22%2C%22shader%22:%22#uicontrol%20invlerp%20normalized%5Cn#uicontrol%2

d-v-b

import dask.array as da
from dask import delayed
import time
import numpy as np
import distributed
from distributed import LocalCluster, Client, performance_report
from datetime import datetime

def blocks(self, index, key_array):
    """

d-v-b

http://neuroglancer-demo.appspot.com/#!%7B%22dimensions%22:%7B%22x%22:%5B1e-9%2C%22m%22%5D%2C%22y%22:%5B1e-9%2C%22m%22%5D%2C%22z%22:%5B1e-9%2C%22m%22%5D%7D%2C%22position%22:%5B24000.5%2C3200.5%2C16684.5%5D%2C%22crossSectionScale%22:50%2C%22projectionScale%22:65536%2C%22layers%22:%5B%7B%22type%22:%22segmentation%22%2C%22source%22:%5B%22n5://https://janelia-cosem.s3.amazonaws.com/jrc_hela-2/jrc_hela-2.n5/labels/endo_seg%22%2C%22precomputed://https://janelia-cosem.s3.amazonaws.com/jrc_hela-2/neuroglancer/mesh/endo_seg%22%5D%2C%22tab%22:%22source%22%2C%22segments%22:%5B%221%22%2C%2210%22%2C%22100%22%2C%221000%22%2C%221001%22%2C%221002%22%2C%221003%22%2C%221004%22%2C%221005%22%2C%221006%22%2C%221007%22%2C%221008%22%2C%221009%22%2C%22101%22%2C%221010%22%2C%221011%22%2C%221012%22%2C%221013%22%2C%221014%22%2C%221015%22%2C%221016%22%2C%221017%22%2C%221018%22%2C%221019%22%2C%22102%22%2C%221020%22%2C%221021%22%2C%221022%22%2C%221023%22%2C%221024%22%2C%221025%22%2C%221026%22%2C%221027%22%2C%221028%22%2C%221029%22%2C%2210

d-v-b

from scipy.ndimage.filters import gaussian_filter
import numpy as np

def generate_mask(array, sigma, percentile, percentile_downsampling=100, threshold=.4):
    smoothed = gaussian_filter(array, sigma=sigma, mode='nearest')
    lower = np.percentile(smoothed.ravel()[::percentile_downsampling], percentile)
    mean = smoothed.mean()
    level = lower + (threshold * (mean - lower))
    mask = (smoothed > level).astype(array.dtype)
    return mask

d-v-b

import numpy as np
import dask.array as da
import numpy as np
import time
import napari

scales = [1,2,4,8,16]
chunks = (128,) * 3
full_size = (8192,8192,8192)

d-v-b

java -Xmx31g -XX:+UseConcMarkSweepGC -cp C:\Users\parkw\.m2\repository\org\janelia\saalfeldlab\n5-utils\0.0.5-SNAPSHOT\n5-utils-0.0.5-SNAPSHOT.jar;C:\Users\parkw\.m2\repository\net\imglib2\imglib2\5.8.0\imglib2-5.8.0.jar;^
C:\Users\parkw\.m2\repository\org\janelia\saalfeldlab\n5\2.1.4\n5-2.1.4.jar;^
C:\Users\parkw\.m2\repository\org\tukaani\xz\1.8\xz-1.8.jar;^
C:\Users\parkw\.m2\repository\org\lz4\lz4-java\1.5.0\lz4-java-1.5.0.jar;^
C:\Users\parkw\.m2\repository\com\google\code\gson\gson\2.8.5\gson-2.8.5.jar;^
C:\Users\parkw\.m2\repository\org\scijava\scijava-common\2.82.0\scijava-common-2.82.0.jar;^
C:\Users\parkw\.m2\repository\org\scijava\parsington\1.0.4\parsington-1.0.4.jar;^
C:\Users\parkw\.m2\repository\org\bushe\eventbus\1.4\eventbus-1.4.jar;^
C:\Users\parkw\.m2\repository\org\apache\commons\commons-compress\1.18\commons-compress-1.18.jar;^
C:\Users\parkw\.m2\repository\org\janelia\saalfeldlab\n5-imglib2\3.4.1\n5-imglib2-3.4.1.jar;^

d-v-b

[hyperlink to viewer](http://neuroglancer-demo.appspot.com/#!%7B%22dimensions%22:%7B%22x%22:%5B1e-9%2C%22m%22%5D%2C%22y%22:%5B1e-9%2C%22m%22%5D%2C%22z%22:%5B1e-9%2C%22m%22%5D%7D%2C%22position%22:%5B21676.611328125%2C2946.377685546875%2C21234.828125%5D%2C%22crossSectionOrientation%22:%5B0%2C0%2C1%2C0%5D%2C%22crossSectionScale%22:63.75196454781246%2C%22projectionOrientation%22:%5B-0.17717696726322174%2C-0.3176162838935852%2C-0.034147508442401886%2C0.9308931827545166%5D%2C%22projectionScale%22:73427.99372841888%2C%22layers%22:%5B%7B%22type%22:%22image%22%2C%22source%22:%22n5://https://janelia-cosem-dev.s3.amazonaws.com/macrophage.n5/fibsem/%22%2C%22shader%22:%22#uicontrol%20float%20upper%20slider%28min=0%2C%20max=1%2C%20step=0.001%2C%20default=1%29%5C%5Cn%5Cn#uicontrol%20float%20lower%20slider%28min=0%2C%20max=1%2C%20step=0.001%2C%20default=0%29%5C%5Cn%5Cnvoid%20main%28%29%20%7BemitGrayscale%28%28clamp%28toNormalized%28getDataValue%28%29%29%2C%20lower%2C%20upper%29%20-%20lower%29%20/%20%28upper%20-%20lower%29%29

d-v-b

Introduction to COSEM

The COSEM (Cellular Organelle Segmentation in Electron Microscopy) project team at Janelia Research campus uses computer vision techniques to scalably detect subcellular structures in datasets generated with next-generation volumetric electron microscopy.

COSEM processes datasets that are acquired by members of the Hess lab at Janelia Research Campus using Focused Ion Beam - Scanning Electron Microscopy (FIB-SEM). FIB-SEM is an electron microscopy technique that enables volumetric imaging of single cells and / or bulk tissue at nanometer isotropic resolution. For more information about FIB-SEM microscopy and its applications, see these publications: Xu et al., Hoffman et al..

FIB-SEM datasets are large (hundreds of gigabytes), dense, and extremely detailed, which poses a challenge for image pr