Histogram in WebGL2 - Video

README.md

Histogram in WebGL2 - Video

view on jsgist

index.css

html, body {
  background-color: #333;
  color: white;
}
video, canvas { border: 1px solid black; margin: 5px; }
#click {
  position: fixed;
  left: 0;
  top: 0;
  width: 100%;
  height: 100%;
  display: flex;
  justify-content: center;
  align-items: center;
}
#click>div {
  padding: 2em;
  background-color: black;
  cursor: pointer;
}
#info {
  background-color: black;
  margin: 0;
  position: fixed;
  right: 0;
  top: 0;
}

index.html

<div id="click">
  <div>click to start</div>
</div>
<pre id="info"></pre>

index.js

import * as twgl from "https://twgljs.org/dist/5.x/twgl-full.module.js";

const histVS = `#version 300 es
uniform sampler2D u_texture;
uniform vec4 u_colorMult;

void main() {
  ivec2 resolution = textureSize(u_texture, 0);
  // based on an id (0, 1, 2, 3 ...) compute the pixel x, y for the source image
  ivec2 pixel = ivec2(gl_VertexID % resolution.x, gl_VertexID / resolution.x);

  // get the pixels but 0 out channels we don't want
  vec4 color = texelFetch(u_texture, pixel, 0) * u_colorMult;

  // add up all the channels. Since 3 are zeroed out we'll get just one channel
  float colorSum = color.r + color.g + color.b + color.a;

  // set the position to be over a single pixel in the 256x1 destination texture
  gl_Position = vec4((colorSum * 255.0 + 0.5) / 256.0 * 2.0 - 1.0, 0.5, 0, 1);

  gl_PointSize = 1.0;
}
`;
const histFS = `#version 300 es
precision mediump float;
out vec4 fragColor;
void main() {
  fragColor = vec4(1);
}
`;
const maxFS = `#version 300 es
precision mediump float;

uniform sampler2D u_texture;
out vec4 fragColor;

void main() {
  vec4 maxColor = vec4(0);

  // we know the texture is 256x1 so just go over the whole thing
  for (int i = 0; i < 256; ++i) {
    // compute centers of pixels
    vec2 uv = vec2((float(i) + 0.5) / 256.0, 0.5);

    // get max value of pixel
    maxColor = max(maxColor, texture(u_texture, uv));
  }

  fragColor = maxColor;
}
`;
const showVS = `#version 300 es
in vec4 position;
void main() {
  gl_Position = position;
}
`;
const showFS = `#version 300 es
precision mediump float;

uniform sampler2D u_histTexture;
uniform vec2 u_resolution;
uniform sampler2D u_maxTexture;

out vec4 fragColor;

void main() {
  // get the max color constants
  vec4 maxColor = texture(u_maxTexture, vec2(0));

  // compute our current UV position
  vec2 uv = gl_FragCoord.xy / u_resolution;

  // Get the history for this color
  // (note: since u_histTexture is 256x1 uv.y is irrelevant
  vec4 hist = texture(u_histTexture, uv);

  // scale by maxColor so scaled goes from 0 to 1 with 1 = maxColor
  vec4 scaled = hist / maxColor;

  // 1 > maxColor, 0 otherwise
  vec4 color = step(uv.yyyy, scaled);

  fragColor = vec4(color.rgb, 1);
}
`;

class QueryManager {
  constructor(gl) {
    this.gl = gl;
    this.ext = gl.getExtension('EXT_disjoint_timer_query_webgl2');
    this.freeQueries = [];
    this.usedQueries = [];
    this.timeElapsed = 'N/A';
  }
  start() {
    const {gl, ext, freeQueries, usedQueries} = this;
    if (!ext) {
      return;
    }
    if (freeQueries.length === 0) {
      freeQueries.push(gl.createQuery());
    }
    const query = freeQueries.pop();
    gl.beginQuery(ext.TIME_ELAPSED_EXT, query);
    usedQueries.push(query);
  }
  end() {
    const {gl, ext, freeQueries, usedQueries} = this;
    if (!ext) {
      return;
    }
    gl.endQuery(ext.TIME_ELAPSED_EXT);
    const available = gl.getQueryParameter(usedQueries[0], gl.QUERY_RESULT_AVAILABLE);
    const disjoint = gl.getParameter(ext.GPU_DISJOINT_EXT);
    if (available && !disjoint) {
      this.timeElapsed = gl.getQueryParameter(usedQueries[0], gl.QUERY_RESULT);
    }
    if (available || disjoint) {
      freeQueries.push(usedQueries.shift());
    }
  }
  get() {
    return this.timeElapsed;
  }
}

async function main() {
  const canvas = document.createElement("canvas");
  canvas.width = 256;
  canvas.height = 120;
  const gl = canvas.getContext("webgl2");
  const ext = gl.getExtension("EXT_color_buffer_float");
  if (!ext) {
    alert("requires EXT_color_buffer_float");
    return;
  }

  const queryManager = new QueryManager(gl);

  const info = document.querySelector('#info');

  const video = document.createElement('video');
  video.crossOrigin = "anonymous";
  video.src = "https://webglsamples.org/color-adjust/sample-video.mp4";
  video.muted = true;
  video.controls = "all";
  video.loop = true;

  await new Promise(resolve => {
    document.addEventListener('click', () => {
      document.querySelector('#click>div').textContent = "loading...";
      video.play();
      const checkVideo = () => {
        if (video.currentTime > 0.1 && video.videoWidth > 0 && video.videoHeight > 0) {
          document.querySelector('#click').style.display = 'none';
          resolve();
        } else {
          setTimeout(checkVideo, 10);
        }
      };
      checkVideo();
    }, {once: true});
  });

  const tex = twgl.createTexture(gl, {
    src: video,
    min: gl.NEAREST,
    mag: gl.NEAREST,
    wrap: gl.CLAMP_TO_EDGE,
  });

  log("video");
  document.body.appendChild(video);
  log("histogram");
  document.body.appendChild(canvas);

  const quadBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
  const histProgramInfo = twgl.createProgramInfo(gl, [histVS, histFS]);

  // make a 256x1 RGBA floating point texture and attach to a framebuffer
  const sumFbi = twgl.createFramebufferInfo(gl, [
    { internalFormat: gl.RGBA32F, minMag: gl.NEAREST, wrap: gl.CLAMP_TO_EDGE, },
  ], 256, 1);
  if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) !== gl.FRAMEBUFFER_COMPLETE) {
    alert("can't render to floating point texture");
  }

  // make a 1x1 pixel RGBA, FLOAT texture attached to a framebuffer
  const maxFbi = twgl.createFramebufferInfo(gl, [
    { internalFormat: gl.RGBA32F, minMag: gl.NEAREST, wrap: gl.CLAMP_TO_EDGE, },
  ], 1, 1);

  const maxProgramInfo = twgl.createProgramInfo(gl, [showVS, maxFS]);
  const showProgramInfo = twgl.createProgramInfo(gl, [showVS, showFS]);

  function render() {
    const startTime = performance.now();
    queryManager.start();

    // update video texture
    gl.bindTexture(gl.TEXTURE_2D, tex);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);

    // Render sum of each color

    // we're going to render a gl.POINT for each pixel in the source image
    // That point will be positioned based on the color of the source image
    // we're just going to render vec4(1,1,1,1). This blend function will
    // mean each time we render to a specific point that point will get
    // incremented by 1.
    gl.blendFunc(gl.ONE, gl.ONE);
    gl.enable(gl.BLEND);
    gl.useProgram(histProgramInfo.program);
    twgl.bindFramebufferInfo(gl, sumFbi);
    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT);

    // render each channel separately since we can only position each POINT
    // for one channel at a time.
    for (var channel = 0; channel < 4; ++channel) {
      gl.colorMask(channel === 0, channel === 1, channel === 2, channel === 3);
      twgl.setUniforms(histProgramInfo, {
        u_texture: tex,
        u_colorMult: [
          channel === 0 ? 1 : 0,
          channel === 1 ? 1 : 0,
          channel === 2 ? 1 : 0,
          channel === 3 ? 1 : 0,
        ],
      });
      gl.drawArrays(gl.POINTS, 0, video.videoWidth * video.videoHeight);
    }
    gl.colorMask(true, true, true, true);
    gl.blendFunc(gl.ONE, gl.ZERO);
    gl.disable(gl.BLEND);

    // render-compute min
    // We're rendering are 256x1 pixel sum texture to a single 1x1 pixel texture

    twgl.bindFramebufferInfo(gl, maxFbi);
    gl.useProgram(maxProgramInfo.program);
    twgl.setBuffersAndAttributes(gl, maxProgramInfo, quadBufferInfo);
    twgl.setUniforms(maxProgramInfo, { u_texture: sumFbi.attachments[0] });
    twgl.drawBufferInfo(gl, quadBufferInfo);

    // render histogram.
    twgl.bindFramebufferInfo(gl, null);
    gl.useProgram(showProgramInfo.program);
    twgl.setUniforms(showProgramInfo, {
      u_histTexture: sumFbi.attachments[0],
      u_resolution: [gl.canvas.width, gl.canvas.height],
      u_maxTexture: maxFbi.attachments[0],
    });
    twgl.drawBufferInfo(gl, quadBufferInfo);
    queryManager.end();

    if (video.requestVideoFrameCallback) {
      video.requestVideoFrameCallback(render);
    } else {
      requestAnimationFrame(render);
    }

    const gpuTime = queryManager.get();
    const elapsedTime = performance.now() - startTime;
    info.textContent = ` js time: ${elapsedTime.toFixed(0.1).padStart('4')}ms
gpu time: ${typeof gpuTime === 'number' ? `${(gpuTime / 1000 / 1000).toFixed(0.1).padStart('4')}ms` : gpuTime}`;
  }
  render();
}

main();

function log() {
  var elem = document.createElement("pre");
  elem.appendChild(document.createTextNode(Array.prototype.join.call(arguments, " ")));
  document.body.appendChild(elem);
}

jsGist.json

{"name":"Histogram in WebGL2 - Video","settings":{},"filenames":["index.html","index.css","index.js"]}