live-cell-boundary-triton.imjoy.html

<docs lang="markdown">
[TODO: write documentation for this plugin.]
</docs>

<config lang="json">
{
  "name": "live-cell-boundary",
  "type": "web-python",
  "version": "0.1.0",
  "description": "[TODO: describe this plugin with one sentence.]",
  "tags": [],
  "ui": "",
  "cover": "",
  "inputs": null,
  "outputs": null,
  "flags": [],
  "icon": "extension",
  "api_version": "0.1.8",
  "env": "",
  "permissions": [],
  "requirements": ["pyotritonclient", "pillow"],
  "dependencies": []
}
</config>

<script lang="python">
from imjoy import api
import io
from PIL import Image
import numpy as np
from js import fetch
from pyotritonclient import get_config, execute_model
import base64
import pyodide
from io import BytesIO
    
async def fetch_image(url, name=None, grayscale=False, size=None):
    response = await fetch(url)
    bytes = await response.arrayBuffer()
    bytes = bytes.to_py()
    buffer = io.BytesIO(bytes)
    buffer.name = name or url.split('?')[0].split('/')[1]
    image = Image.open(buffer).convert('L')
    if grayscale:
        image = image.convert('L')
    if size:
        image = image.resize(size=size)
    image = np.array(image)
    return image

def encode_image(image):
    image = Image.fromarray(image)
    buffered = BytesIO()
    image.save(buffered, format="PNG")
    img_str = 'data:image/png;base64,' + base64.b64encode(buffered.getvalue()).decode('ascii')
    return img_str

class ImJoyPlugin():
    def setup(self):
        api.log('initialized')
    
    async def run_inference(self):
        # run inference
        results = await execute_model([self.input_image[None, :, :].astype('float32'), {}],
                                      server_url='https://ai.imjoy.io/triton',
                                      model_name='bioimageio-live-cell-segmentation-boundary-model',
                                      decode_bytes=True)
        mask = results['output_0'][1, :, :] * 255
        await self.viewer.view_image(encode_image(mask.astype('uint8')), name="mask")

    async def run(self, ctx):
        viewer = await api.createWindow({"src": "https://kaibu.org/#/app", "fullscreen": True})
        widget_config = {
            "_rintf": True,
            "name": "Control",
            "type": "control",
            "elements": [
                {
                    "type": "button",
                    "label": "Run inference",
                    "callback": self.run_inference,
                },
            ]
        }
        await viewer.add_widget(pyodide.to_js(widget_config, dict_converter=Object.fromEntries))
        

        image = await fetch_image('https://zenodo.org/api/files/a6d477fe-9412-4064-b7e6-67f057fec920/sample_input_0.tif', grayscale=True)
        await viewer.view_image(encode_image(image), name="image")
        self.input_image = image
        self.viewer = viewer
        

api.export(ImJoyPlugin())
</script>