8thWall video background with post-processing

threePipelineModule.js

const threePipelineModule = () => {
  let scene3
  let _videoWidth
  let _videoHeight
  let fxaaPass = {}
  return {
    name    : 'customthreemodule',
    onStart : ({ canvas, canvasWidth, canvasHeight, videoWidth, videoHeight, GLctx }) => {
      const scene = new window.THREE.Scene()
      const camera = new window.THREE.PerspectiveCamera(
        60.0,  /* initial field of view; will get set based on device info later. */
        canvasWidth / canvasHeight,
        0.01,
        1000.0
      )
      scene.add(camera)
      _videoWidth = videoWidth
      _videoHeight = videoHeight

      const renderer = new window.THREE.WebGLRenderer({
        canvas,
        context   : GLctx,
        alpha     : false,
        antialias : true,
      })
      renderer.autoClear = false
      renderer.setSize(canvasWidth, canvasHeight)
      renderer.shadowMap.enabled = true
      renderer.shadowMap.type = window.THREE.PCFShadowMap
      renderer.gammaOutput = true
      renderer.gammaFactor = 2.2
      renderer.toneMapping = window.THREE.Uncharted2ToneMapping
      renderer.toneMappingExposure = 0.8

      // Background texture
      const backgroundTexture = new window.THREE.Texture()
      backgroundTexture.encoding = window.THREE.sRGBEncoding
      backgroundTexture.minFilter = window.THREE.LinearFilter
      backgroundTexture.magFilter = window.THREE.LinearFilter
      backgroundTexture.flipY = false
      backgroundTexture.format = window.THREE.RGBFormat

      // Then create a plane textured with the webcam video.
      const backgroundPlane = new window.THREE.Mesh(
        new window.THREE.PlaneBufferGeometry(2, 2),
        new window.THREE.MeshBasicMaterial({ map: backgroundTexture, side: window.THREE.DoubleSide })
      )

      // The webcam video plane shouldn't care about the z-buffer.
      backgroundPlane.material.depthTest = false
      backgroundPlane.material.depthWrite = false

      // Create a camera and a scene for the webcam video plane and
      // add the camera and the webcam video plane to the scene.
      const videoBackgroundCamera = new window.THREE.OrthographicCamera(-1, 1, -1, 1, -1, 1)
      const videoBackgroundScene = new window.THREE.Scene()
      videoBackgroundScene.add(backgroundPlane)
      videoBackgroundScene.add(videoBackgroundCamera)

      const composer = new EffectComposer(renderer)
      const videoBackgroundRenderPass = new RenderPass(videoBackgroundScene, videoBackgroundCamera)
      composer.addPass(videoBackgroundRenderPass)
      const renderPass = new RenderPass(scene, camera)
      renderPass.clear = false
      composer.addPass(renderPass)

      fxaaPass = new ShaderPass(FXAAShader)
      const pixelRatio = renderer.getPixelRatio()
      fxaaPass.material.uniforms.resolution.value.x = 1 / (canvasWidth * pixelRatio)
      fxaaPass.material.uniforms.resolution.value.y = 1 / (canvasHeight * pixelRatio)
      fxaaPass.renderToScreen = true
      composer.addPass(fxaaPass)

      composer.setSize(canvasWidth, canvasHeight)

      scene3 = { scene, camera, renderer, composer, backgroundTexture }
    },
    onUpdate: ({ processCpuResult }) => {
      if (!processCpuResult.reality) {
        return
      }
      const { rotation, position, intrinsics, realityTexture } = processCpuResult.reality
      const { camera, scene, renderer, backgroundTexture } = scene3
      if (realityTexture) {
        const texProps = renderer.properties.get(backgroundTexture)
        // eslint-disable-next-line no-underscore-dangle
        texProps.__webglTexture = realityTexture
        backgroundTexture.needsUpdate = true
        const { x:canvasWidth, y:canvasHeight } = scene3.renderer.getSize()
        const canvasAspect = canvasWidth / canvasHeight
        const imageAspect = _videoWidth / _videoHeight
        const aspect = imageAspect / canvasAspect
        backgroundTexture.offset.x = aspect > 1 ? (1 - 1 / aspect) / 2 : 0
        backgroundTexture.repeat.x = aspect > 1 ? 1 / aspect : 1
        backgroundTexture.offset.y = aspect > 1 ? 0 : (1 - aspect) / 2
        backgroundTexture.repeat.y = aspect > 1 ? 1 : aspect
      }

      for (let i = 0; i < 16; i++) {
        camera.projectionMatrix.elements[i] = intrinsics[i]
      }

      // Default 8th wall three module isn't doing this
      camera.projectionMatrixInverse.getInverse(camera.projectionMatrix)

      for (let i = 0; i < 16; i++) {
        camera.projectionMatrix.elements[i] = intrinsics[i]
      }
      if (rotation) {
        camera.setRotationFromQuaternion(rotation)
      }
      if (position) {
        camera.position.set(position.x, position.y, position.z)
      }
    },
    onCanvasSizeChange: ({ canvasWidth, canvasHeight }) => {
      scene3.renderer.setSize(canvasWidth, canvasHeight)
      scene3.composer.setSize(canvasWidth, canvasHeight)
      const pixelRatio = scene3.renderer.getPixelRatio()
      fxaaPass.material.uniforms.resolution.value.x = 1 / (canvasWidth * pixelRatio)
      fxaaPass.material.uniforms.resolution.value.y = 1 / (canvasHeight * pixelRatio)
    },
    onRender: () => {
      const { renderer, scene, camera, composer } = scene3
      renderer.clear()

      if (composer) {
        console.log('using composer')
        composer.render()
      } else {
        console.log('usual renderer')
        renderer.render(scene, camera)
      }
    },
    xrScene: () => scene3,
  }
}