webxr-heldheld-ar-cached-framebuffers-proposal.js

// Reference: https://github.com/immersive-web/webxr/issues/849
// Author: Andrew Johnon
//         ajohnson@draster.com
//
// PROPOSAL:
// For handheld AR, allow for camera framebuffer caching and later
// retrieval+render.
//
// This facilitates handheld AR usecases where a pose's transform data is sent to
// a server, it gets processed in some way, then after a small period of time
// (10s of milliseconds), some information is returned back to the client where
// it could be composited against the framebuffer corresponding with the
// original pose. This would introduce an additional delay in screen display of
// the camera feed, but this additional delay can be acceptable in handheld AR
// scenarios given the value it can add, such as remote rendering of very high
// density assets that wouldn't be possible on mobile hardware (billion+
// triangles, proof of concept here: https://youtu.be/BQM9WyrXie4?t=415 @ 6:55).
//
// Any new API surfaces are tagged with "NEW API"

let sess;

async function startAr() // Entrypoint to AR
{
  const myWebGlRenderer = initMyGLRenderer();
  
  // NEW API: The "framebuffer-caching" feature may be requested via requiredFeatures / optionalFeatures.
  sess = await xr.requestSession("immersive-ar", { requiredFeatures: ["framebuffer-caching"] });

  // NEW API: The enableFramebufferCaching flag allows usage of (and causes
  // expectation of?) baseLayer.cacheFramebuffer() /
  // baseLayer.getCachedFramebuffer() rather than baseLayer.framebuffer.
  // Perhaps this is not necessary since the first usage of
  // `baseLayer.cacheFramebuffer` could signal this to the webxr implementation,
  // but I'm imagining the implementation may need to know ahead of time that
  // the pose retrieval and the pose's corresponding framebuffer usage will be
  // decoupled.
  await sess.updateRenderState({"baseLayer": new XRWebGLLayer(sess, myWebGlRenderer.glCtx),
                                "enableFramebufferCaching": true});

  sess.requestAnimationFrame(this.xrUpdate);
}

function xrUpdate(time, frame)
{
  sess.requestAnimationFrame(this.xrUpdate);
  let pose = frame.getViewerPose(...);
  const view = pose.views[0];

  // NEW API: baseLayer.cacheFramebuffer() caches the framebuffer and returns an
  // integer reference to it.  Note this is only an integer handle and allows no
  // access to the underlying framebuffer data.  A user camera data request
  // should therefore not be required.
  const cachedFramebufferId = sess.baseLayer.cacheFramebuffer();

  // requestFrameRenderFromServer(): This is not a part of webxr, but is a
  // hypothetical user function that would request a scene be rendered on a
  // server given a view transform / projection.  cachedFramebufferId is also
  // passed to the server so that it can be attached to the corresponding frame
  // data that gets rendered and sent back to the client.
  requestFrameRenderFromServer(cachedFramebufferId, view.projectionMatrix, view.transform);
}

// gotFrameFromServer(): This is not a part of webxr, but is a hypothetical user
// callback that occurs when receiving a rendered video frame from a server.
//
// The server has tracked and provided back to us the cachedFramebufferId
// attribute in the renderedFrame object, so we can reference the corresponding
// camera framebuffer using `baseLayer.getCachedFramebuffer()`.
//
// Note the video would normally be bound to a GL texture and there are some
// other compositing tricks that need doing, but I skipped those implementation
// details for the sake of brevity.
function gotFrameFromServer(renderedFrame)
{
  // NEW API: baseLayer.getCachedFramebuffer() retrieves a cached framebuffer
  // given the cachedFramebufferId.
  const cachedFramebuffer = sess.baseLayer.getCachedFramebuffer(renderedFrame.cachedFramebufferId);

  // If the amount of time between requestFrameRenderFromServer and
  // gotFrameFromServer is greater than the what the underlying camera delay
  // (probably ring) buffer can accomodate, the cached framebuffer may no longer
  // be available.
  if (!cachedFramebuffer)
  {
    console.log("WARNING: cached framebuffer not found!");
    return; // There is no valid framebuffer, so do not attempt to render the frame.
  }

  // NEW API (kind of): We're passing the cached framebuffer rather than
  // baseLayer.framebuffer
  myWebGlRenderer.glCtx.bindFramebuffer(myWebGlRenderer.glCtx.FRAMEBUFFER, cachedFramebuffer);

  myWebGlRenderer.render();
}

startAr();