NiiVue SAM WIP

test.js

// 1. hide crosslines
nv.setCrosshairColor([0,0,0,0]);
nv.opts.crosshairWidth=0;
nv.updateGLVolume();


// 2. grab REAL pixels
c = document.getElementById('viewer')
ctx = c.getContext("webgl2")

nv.refreshDrawing() // draw call
pixels = new Uint8Array(ctx.drawingBufferWidth * ctx.drawingBufferHeight * 4);
ctx.readPixels(
  0, 
  0, 
  ctx.drawingBufferWidth, 
  ctx.drawingBufferHeight, 
  ctx.RGBA, 
  ctx.UNSIGNED_BYTE, 
  pixels);


offscreen = document.createElement('canvas'),
offscreen.width = ctx.drawingBufferWidth;
offscreen.height = ctx.drawingBufferHeight;

offctx = offscreen.getContext('2d');

// create imageData object
imgdata = offctx.createImageData(offscreen.width, offscreen.height);

pxdata = imgdata.data;

for (var i =0; i<pxdata.length;i++) {
    
  pxdata[i] = pixels[i];

}

// update canvas with new data
offctx.putImageData(imgdata, 0, 0);

base64 = offscreen.toDataURL('image/png')
base64 = base64.replace("data:image/png;base64,","")
realpixels = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));



// 3. LATER: make selection interactive
// see https://github.com/niivue/niivue/issues/596
x=200;
y=200;
width=100;
height=100;
nv.drawSelectionBox([x,y,width,height]);


// 4. get image embedding
endpoint = 'https://model-zoo.metademolab.com/predictions/segment_everything_box_model';        

xhr = new XMLHttpRequest();
xhr.open("POST", endpoint);
xhr.onreadystatechange = function () {
  if (xhr.readyState === 4) {

    embedding = JSON.parse(xhr.response);

    // LOAD ONNX RUNTIME
    const script = document.createElement("script")
    script.type = "text/javascript"
    script.src = "https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js"
    document.head.appendChild(script)
    eval(script)

  }
}
xhr.send(realpixels);


// 5. SEGMENT
// same as the other bookmarklet