Slicer, open3d, azure kinect, MRHead

facehead.py

"""

pip_install("open3d")

rename "/c/Program\ Files/Azure\ Kinect\ SDK\ v1.4.0/tools/k4a.dll" by changing 1.4 to 1.2

path="c:/pieper/facenav/facenav.py"
exec(open(path).read())


"""

import numpy
import open3d

class KinectStream:

    def __init__(self, config, device, align_depth_to_color, update_period=10):
        self.align_depth_to_color = align_depth_to_color
        self.update_period = update_period
        self.near = 400
        self.far = 1000
        self.running = False
        self.depthVolume = None
        self.colorVolume = None
        self.model = None

        self.sensor = open3d.io.AzureKinectSensor(config)
        if not self.sensor.connect(device):
            raise RuntimeError('Failed to connect to sensor')

    def createModel(self):
        # Create model node
        planeSource = vtk.vtkPlaneSource()
        shape = self.depthArray.shape[1:]
        planeSource.SetResolution(shape[1]-1, shape[0]-1)
        planeSource.SetOrigin(0.0, 0.0, 0.0)
        planeSource.SetPoint2(shape[0], 0.0, 0.0)
        planeSource.SetPoint1(0.0, shape[1], 0.0)
        self.model = slicer.modules.models.logic().AddModel(planeSource.GetOutputPort())

        # Tune display properties
        modelDisplay = self.model.GetDisplayNode()
        modelDisplay.SetColor(0.5,0.5,0.5) 
        modelDisplay.SetBackfaceCulling(0)

        colorDataArray = vtk.vtkUnsignedCharArray()
        colorDataArray.SetName("Color")
        colorDataArray.SetNumberOfComponents(3)
        colorDataArray.SetNumberOfTuples(shape[0]*shape[1])
        self.model.GetPolyData().GetPointData().AddArray(colorDataArray)

        self.model.GetDisplayNode().SetActiveScalarName("Color")
        self.model.GetDisplayNode().SetScalarVisibility(True)
        self.model.GetDisplayNode().SetAndObserveColorNodeID('vtkMRMLColorTableNodeGrey')

    def updateModel(self):
        colorArray = slicer.util.arrayFromModelPointData(self.model, "Color")
        colorArray.reshape(self.colorArray.shape)[:] = self.colorArray
        slicer.util.arrayFromModelPointDataModified(self.model, "Color")
        points = slicer.util.arrayFromModelPoints(self.model)
        points[:,2] = numpy.clip(self.depthArray.flatten(), self.near, self.far)
        p = points[:,2]
        p[p==self.near] = self.far
        slicer.util.arrayFromModelPointsModified(self.model)

    def step(self):
        if not self.running:
            return
        self.rgbd = self.sensor.capture_frame(self.align_depth_to_color)
        if self.rgbd is not None:
            self.depthArray = numpy.asarray(self.rgbd.depth)
            newshape = list(self.depthArray.shape)
            newshape.insert(0,1)
            self.depthArray = self.depthArray.reshape(newshape)
            self.colorArray = numpy.asarray(self.rgbd.color)
            newshape = list(self.colorArray.shape)
            newshape.insert(0,1)
            self.colorArray = self.colorArray.reshape(newshape)
            ijkToRAS = numpy.diag([1.0, -1.0, 1.0, 1.0])
            if not self.depthVolume or not self.colorVolume:
                self.depthVolume = slicer.util.addVolumeFromArray(self.depthArray, ijkToRAS=ijkToRAS, name="depth")
                self.colorVolume = slicer.util.addVolumeFromArray(self.colorArray, ijkToRAS=ijkToRAS, name="color",
                                                                    nodeClassName="vtkMRMLVectorVolumeNode")
                self.createModel()
            else:
                slicer.util.updateVolumeFromArray(self.depthVolume, self.depthArray)
                slicer.util.updateVolumeFromArray(self.colorVolume, self.colorArray)
                self.updateModel()
        qt.QTimer.singleShot(self.update_period, self.step)

    def go(self):
        self.running = True
        self.step()

    def stop(self):
        self.running = False

config = open3d.io.AzureKinectSensorConfig()
device = 0
align_depth_to_color = True
kinect = KinectStream(config, device, align_depth_to_color)
kinect.go()