BodyPix を getUserMedia に注入してみる

body-pix-injector.js

'use strict'

;(async () => {

  const handleSelectImage = (input, select) => {
    input.addEventListener('change', e => {
      const files = [...input.files]
      const image = files.filter(f => f.type.startsWith('image/'))
      if (image.length) {
        const file = image[0]
        e.preventDefault()
        select(file)
        input.value = ''
      }
    })
  }

  const getImageItem = transfer => {
    const items = [...transfer.items]
    const image = items.filter(i => i.type.startsWith('image/'))
    return image.length > 0 ? image[0] : null
  }

  const handleDropImage = (scope, condition, drag, drop) => {

    scope.addEventListener('dragenter', e => {
      if (condition()) {
        e.preventDefault()
        e.stopPropagation()
        drag(true)
      }
    })
    scope.addEventListener('dragleave', e => {
      if (condition()) {
        e.preventDefault()
        e.stopPropagation()
        drag(false)
      }
    })
    scope.addEventListener('dragover', e => {
      if (condition()) {
        e.preventDefault()
        e.stopPropagation()
        e.dataTransfer.effectAllowed = 'copy'
        e.dataTransfer.dropEffect = getImageItem(e.dataTransfer) ? 'copy' : 'none'
      }
    })
    scope.addEventListener('drop', e => {
      if (condition()) {
        e.preventDefault()
        e.stopPropagation()
        const item = getImageItem(e.dataTransfer)
        if (item) {
          drag(false)
          drop(item.getAsFile())
        }
      }
    })

  }

  const handlePasteImage = (condition, paste) => {
    document.addEventListener('paste', e => {
      if (condition()) {
        const item = getImageItem(e.clipboardData)
        if (item) {
          paste(item.getAsFile())
        }
      }
    })
  }

  const movable = (target, grip) => {

    let state = {
      move: false,
    }

    grip.addEventListener('mousedown', e => {
      e.stopPropagation()
      const bounds = target.getBoundingClientRect()
      const dx = bounds.left - e.clientX
      const dy = bounds.top - e.clientY
      state = {
        move: true,
        dx,
        dy,
      }
      e.preventDefault()
    })

    grip.addEventListener('mouseup', e => {
      e.stopPropagation()
      state = {
        move: false,
      }
      e.preventDefault()
    })

    grip.addEventListener('mousemove', e => {
      e.stopPropagation()
      if (!state.move) {
        return
      }
      const x = e.clientX + state.dx
      const y = e.clientY + state.dy
      target.style.setProperty('left', x + 'px', 'important')
      target.style.setProperty('top', y + 'px', 'important')
      target.style.setProperty('right', 'auto', 'important')
      target.style.setProperty('bottom', 'auto', 'important')
      e.preventDefault()
    })

    grip.addEventListener('pointerdown', e => {
      e.target.setPointerCapture?.(e.pointerId)
    }, {passive: true, capture: true})

    grip.addEventListener('pointerup', e => {
      e.target.releasePointerCapture?.(e.pointerId)
    }, {passive: true, capture: true})

  }

  //

  const loadBodyPix = async () => {

    // https://github.com/tensorflow/tfjs-models/tree/master/body-pix

    const injectScript = (url, detector, initializer = null) => {
      if (detector?.()) {
        console.info(`bodypix-injector: {url} loaded.`)
        return Promise.resolve()
      }
      return new Promise((resolve, reject) => {
        const script = document.createElement('script')
        script.addEventListener('load', e => resolve(script))
        script.addEventListener('error', e => reject(script))
        // script.async = true
        initializer?.(script)
        document.head.appendChild(script)
        script.src = url
      })
    }

    await injectScript('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@1.2', () => window.tf)
    await injectScript('https://cdn.jsdelivr.net/npm/@tensorflow-models/body-pix@2.0', () => window.bodyPix)

    const net = await bodyPix.load({
      // architecture: 'MobileNetV1', // ResNet50
      // outputStride: 16,
      // multiplier: 0.75,
      // quantBytes: 2
    });

    console.info('bodypix-injector: BodyPix model loaded.')

    return net

  }

  const createControlPanel = selectImageFile => {

    const render = () => {
      return `
        <style>
          @import 'https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css';          
          .bodypix-injector {
            font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", "Liberation Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
            font-size: 1rem;
            font-weight: 400;
            line-height: 1.5;
            color: #212529;
            background-color: #fff;
            -webkit-text-size-adjust: 100%;
            -webkit-tap-highlight-color: transparent
          }
  
          .form-range {
            margin-top: .375rem;
            margin-bottom: .375rem;
          }
          
          .form-control-color {
            border: 0;
          }

          .dropping {
            2px solid blue;
          }
          .dropping * {
            pointer-events: none;
          }  
  
        </style>
        
        <aside class="bodypix-injector">
  
          <a href="#modeless" data-toggle="modeless" title="Configure BodyPix" class="btn btn-outline-primary">
            <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-person-lines-fill" viewBox="0 0 16 16">
              <path d="M6 8a3 3 0 1 0 0-6 3 3 0 0 0 0 6zm-5 6s-1 0-1-1 1-4 6-4 6 3 6 4-1 1-1 1H1zM11 3.5a.5.5 0 0 1 .5-.5h4a.5.5 0 0 1 0 1h-4a.5.5 0 0 1-.5-.5zm.5 2.5a.5.5 0 0 0 0 1h4a.5.5 0 0 0 0-1h-4zm2 3a.5.5 0 0 0 0 1h2a.5.5 0 0 0 0-1h-2zm0 3a.5.5 0 0 0 0 1h2a.5.5 0 0 0 0-1h-2z"/>
            </svg>
          </a>
    
          <div id="modeless" class="position-fixed d-none" role="dialog" style="width: 500px; left: 1rem; bottom: 3rem">
            <div class="modal-dialog modal-lg" style="margin: 0">
              <div class="modal-content">
                <div class="modal-header">
                  <h3>BodyPix injector</h3>
                  <button type="button" data-dismiss="modeless" class="btn-close" aria-label="Close"></button>
                </div>
                <div class="modal-body">
                  <form id="configuration" onsubmit="return false">
  
                    <fieldset>
                      <div class="row">
                        <h4 class="col-4 fw-normal">Effect</h4>
                        <div class="col-8">
                          <div class="form-check form-check-inline">
                            <input class="form-check-input" type="radio" name="effect" id="effect-bokeh" value="bokeh" checked>
                            <label class="form-check-label" for="effect-bokeh">Bokeh</label>
                          </div>
                          <div class="form-check form-check-inline">
                            <input class="form-check-input" type="radio" name="effect" id="effect-mask" value="mask">
                            <label class="form-check-label" for="effect-mask">Image</label>
                          </div>
                          <div class="form-check form-check-inline">
                            <input class="form-check-input" type="radio" name="effect" id="effect-off" value="off">
                            <label class="form-check-label" for="effect-off">Off</label>
                          </div>
                        </div>
                      </div>
                    </fieldset>
        
                    <hr>
        
                    <fieldset>
                      <legend>Bokeh effect</legend>
                      <div class="row">
                        <label class="col-4 col-form-label" for="bokeh-edge-blur">Edge blur</label>
                        <div class="col-8">
                          <input type="range" class="form-range" id="bokeh-edge-blur" min="1" max="20" value="3" placeholder="Edge blur">
                        </div>
                      </div>
                      <div class="row">
                        <label class="col-4 col-form-label" for="bodypix-bokeh-injector-bokeh-background-blur">Background blur</label>
                        <div class="col-8">
                          <input type="range" class="form-range" id="bokeh-background-blur" min="0" max="20" value="3" placeholder="Background blur">
                        </div>
                      </div>
                    </fieldset>
  
                    <hr>
  
                    <fieldset>                              
                      <legend>Image effect</legend>
                      <div class="row">
                        <label class="col-4 col-form-label" for="image-background-image-picker">Background image</label>
                        <div class="col-8">
                          <div class="position-relative d-inline-block align-middle form-control form-control-color">
                            <img id="image-background-image" class="position-absolute top-50 start-50 translate-middle border rounded" style="max-width: 2.25rem; max-height: 2.25rem" alt="" title="Current image">
                          </div>
                          <input type="file" accept="image/*" class="d-none form-control" id="image-background-image-picker" placeholder="Background image">
                          <button class="btn btn-sm btn-outline-secondary" type="button" id="pick-image">Select</button>
                          <button class="btn btn-sm btn-outline-secondary" type="button" id="clear-image">Clear</button>
                        </div>
                      </div>
                      <div class="row">
                        <label class="col-4 col-form-label" for="image-background-color">Background color</label>
                        <div class="col-8">
                          <input type="color" class="form-control form-control-color" id="image-background-color" value="#e0e0e0" placeholder="Background color">
                        </div>
                      </div>
                      <div class="row">
                        <label class="col-4 col-form-label" for="image-edge-blur">Edge blur</label>
                        <div class="col-8">
                          <input type="range" class="form-range" id="image-edge-blur" min="1" max="20" value="3" placeholder="Edge blur">
                        </div>
                      </div>
                      <div class="row">
                        <label class="col-4 col-form-label" for="bodypix-bokeh-injector-image-background-blur">Background blur</label>
                        <div class="col-8">
                          <input type="range" class="form-range" id="image-background-blur" min="0" max="20" value="3" placeholder="Background blur">
                        </div>
                      </div>
                    </fieldset>
  
                    <hr>
  
                    <fieldset>
                      <legend>Default camera</legend>
                      <div class="row">
                        <label class="col-4 col-form-label" for="default-camera">Default camera</label>
                        <div class="col-8">
                            <div class="input-group">
                              <select class="form-select d-none" id="default-camera" placeholder="Default camera">
                                  <option value="">Auto</option>
                              </select>
                              <button class="btn btn-outline-secondary" type="button" id="query-devices">Select</button>
                            </div>
                        </div>
                      </div>
                    </fieldset>
  
                    <hr>
       
                    <details>
                      <summary>More tweaks</summary>
                    
                      <fieldset>
                        <legend>Person segmentation</legend>
                        <div class="row">
                          <label class="col-4 col-form-label" for="segmentation-internal-resolution">Internal resolution</label>
                          <div class="col-8">
                            <input type="range" class="form-range" id="segmentation-internal-resolution" min="0.25" max="1.0" value="0.5" step="0.25" placeholder="Internal resolution">
                          </div>
                        </div>
                        <div class="row">
                          <label class="col-4 col-form-label" for="segmentation-segmentation-threshold">Segmentation threshold</label>
                          <div class="col-8">
                            <input type="range" class="form-range" id="segmentation-segmentation-threshold" min="0.0" max="1.0" value="0.7" step="0.01" placeholder="Segmentation threshold">
                          </div>
                        </div>
                        <div class="row">
                          <label class="col-4 col-form-label" for="segmentation-max-detections">Max detections</label>
                          <div class="col-8">
                            <input type="range" class="form-range" id="segmentation-max-detections" min="1" max="50" value="10" placeholder="Max detections">
                          </div>
                        </div>
                        <div class="row">
                          <label class="col-4 col-form-label" for="segmentation-score-threshold">Score threshold</label>
                          <div class="col-8">
                            <input type="range" class="form-range" id="segmentation-score-threshold" min="0.0" max="1.0" value="0.3" step="0.01" placeholder="Score threshold">
                          </div>
                        </div>
                        <div class="row">
                          <label class="col-4 col-form-label" for="segmentation-nms-radius">Non-maximum suppression Radius</label>
                          <div class="col-8">
                            <input type="range" class="form-range" id="segmentation-nms-radius" min="1" max="200" value="20" placeholder="Non-maximum suppression">
                          </div>
                        </div>
                      </fieldset>
  
                    </details>
  
                  </form>
                </div>
    
                <div class="modal-footer">
                  <a class="me-auto" href="https://blog.tensorflow.org/2019/11/updated-bodypix-2.html">BodyPix</a>
<!--                  <button type="button" id="test" class="btn btn-outline-danger me-auto">Test</a>-->
                  <button type="reset" form="configuration" class="btn btn-outline-secondary">Reset</a>
                  <button type="button" data-dismiss="modeless" class="btn btn-outline-primary">Close</a>
                </div>
    
              </div>
            </div>
          </div>
  
        </aside>
      `

    }

    const panel = document.createElement('div')
    panel.className = 'bodypix-injector'
    panel.style.position = 'fixed'
    panel.style.left = '16px'
    panel.style.bottom = '16px'

    const shadow = panel.attachShadow({mode: 'open'})
    shadow.innerHTML = render()

    const enableModelessInvokers = () => {

      const modeless = shadow.querySelector('#modeless')

      shadow.querySelectorAll('[data-dismiss=modeless]').forEach(e => {
        e.addEventListener('click', e => {
          modeless.classList.add('d-none')
          e.preventDefault()
        })
      })
      shadow.querySelectorAll('[data-toggle=modeless]').forEach(e => {
        e.addEventListener('click', e => {
          modeless.classList.toggle('d-none')
          e.preventDefault()
        })
      })

      shadow.querySelectorAll('input[type=range]').forEach(e => {
        e.title = e.value
        e.addEventListener('input', e => {
          e.currentTarget.title = e.currentTarget.value
        })
      })

    }

    const enableDeviceList = () => {

      // device list
      const deviceList = shadow.querySelector('#default-camera')
      const deviceLister = shadow.querySelector('#query-devices')

      const group = key => (groups, e) => {
        const k = key(e)
        return {...groups, [k]: [...(groups[k] || []), e]}
      }

      const listDevices = async () => {

        const devices = await navigator.mediaDevices.enumerateDevices()
        const videos = devices.filter(d => d.kind === 'videoinput')

        if (videos.length === 0 || videos[0].deviceId === '') {
          // query permission
          const stream = await navigator.mediaDevices.getUserMedia({video: true})
          stream.getTracks().forEach(t => t.stop())
          return listDevices()
        }

        const el = (name, attrs = {}, content = null) => {
          const e = document.createElement(name)
          Object.entries(attrs).forEach(([name, value]) => e.setAttribute(name, value))
          if (content !== null) {
            e.append(content)
          }
          return e
        }

        const groups = videos.reduce(group(d => d.groupId), {})

        const options = Object.entries(groups).map(([g, list]) => {
          if (list.length > 1) {
            return el('optgroup', {'data-group': g}, list.map(d => el('option', {value: d.deviceId}, d.label)))
          } else {
            return el('option', {'data-group': g, value: list[0].deviceId}, list[0].label)
          }
        })

        deviceList.innerHTML = ''
        deviceList.append(el('option', {value: ''}, 'Auto'), ...options)
        deviceList.classList.remove('d-none')
        deviceLister.textContent = 'Reload'

      }

      deviceLister.addEventListener('click', e => {
        listDevices()
      })

    }

    const enableImagePicker = () => {

      const modeless = shadow.querySelector('#modeless')
      const visible = () => !modeless.classList.contains('d-none')

      const showDragging = active => {
        modeless.classList[active ? 'add' : 'remove']('dropping')
      }

      const picker = shadow.querySelector('#image-background-image-picker')

      // select button
      shadow.querySelector('#pick-image').addEventListener('click', e => {
        picker.click()
      })

      // clear button
      shadow.querySelector('#clear-image').addEventListener('click', e => {
        selectImageFile(null)
      })

      // support file selector
      handleSelectImage(picker, selectImageFile)

      // support file drop during dialog shown
      handleDropImage(modeless, visible, showDragging, selectImageFile)

      // support paste operation during dialog shown
      handlePasteImage(visible, selectImageFile)

    }

    enableModelessInvokers()
    enableDeviceList()
    enableImagePicker()

    const modeless = shadow.querySelector('#modeless')

    movable(modeless, modeless.querySelector('.modal-header'))

    document.body.append(panel)

    return shadow

  }

  let currentBackgroundImageUrl = null
  let currentBackgroundImage = null

  const changeBackgroundImageFile = file => {
    currentBackgroundImageUrl && URL.revokeObjectURL(currentBackgroundImageUrl)
    currentBackgroundImageUrl = file ? URL.createObjectURL(file) : null
    if (currentBackgroundImageUrl) {
      const image = new Image()
      image.crossOrigin = 'anonymous'
      image.onload = e => {
        currentBackgroundImage = image
      }
      image.src = currentBackgroundImageUrl
      imageBackgroundImage.src = currentBackgroundImageUrl // preview
    } else {
      currentBackgroundImage = null
      imageBackgroundImage.src = null // preview
    }
  }

  const getBackgroundImage = () => {
    return currentBackgroundImage
  }

  const panel = createControlPanel(changeBackgroundImageFile)

  const effectBokeh = panel.querySelector('#effect-bokeh')
  const effectMask = panel.querySelector('#effect-mask')
  const effectOff = panel.querySelector('#effect-off')

  const imageBackgroundImage = panel.querySelector('#image-background-image')
  const imageBackgroundImagePicker = panel.querySelector('#image-background-image-picker')
  const imageBackgroundColor = panel.querySelector('#image-background-color')
  const imageEdgeBlur = panel.querySelector('#image-edge-blur')
  const imageBackgroundBlur = panel.querySelector('#image-background-blur')

  const bokehEdgeBlur = panel.querySelector('#bokeh-edge-blur')
  const bokehBackgroundBlur = panel.querySelector('#bokeh-background-blur')

  const segmentationInternalResolution = panel.querySelector('#segmentation-internal-resolution')
  const segmentationSegmentationThreshold = panel.querySelector('#segmentation-segmentation-threshold')
  const segmentationMaxDetections = panel.querySelector('#segmentation-max-detections')
  const segmentationScoreThreshold = panel.querySelector('#segmentation-score-threshold')
  const segmentationNmsRadius = panel.querySelector('#segmentation-nms-radius')

  const defaultCamera = panel.querySelector('#default-camera')

  //
  //
  //

  const collectImageOptions = () => {
    return {
      edgeBlurAmount: +imageEdgeBlur.value,
      backgroundBlurAmount: +imageBackgroundBlur.value,
    }
  }

  const collectBokehOptions = () => {
    return {
      edgeBlurAmount: +bokehEdgeBlur.value,
      backgroundBlurAmount: +bokehBackgroundBlur.value,
    }
  }

  const collectSegmentationOptions = () => {
    return {
      internalResolution: +segmentationInternalResolution.value,
      segmentationThreshold: +segmentationSegmentationThreshold.value,
      maxDetections: +segmentationMaxDetections.value,
      scoreThreshold: +segmentationScoreThreshold.value,
      nmsRadius: +segmentationNmsRadius.value,
    }
  }

  const offscreens = {}
  const prepareOffscreen = (name, size) => {
    const canvas = offscreens[name] || (offscreens[name] = document.createElement('canvas'))
    canvas.width = size.width
    canvas.height = size.height
    return canvas
  }

  const blur = (name, image, amount) => {
    const canvas = prepareOffscreen('name', image)
    const g = canvas.getContext('2d')
    g.save()
    g.clearRect(0, 0, canvas.width, canvas.height)
    g.filter = `blur(${amount}px)`
    g.drawImage(image, 0, 0, canvas.width, canvas.height)
    g.restore()
    return canvas
  }

  const net = await loadBodyPix()

  const copyFrame = async (source, destination) => {
    const g = destination.getContext('2d')
    g.save()
    g.drawImage(source, 0, 0, destination.width, destination.height)
    g.restore()
  }

  const bokehFrame = async (source, destination) => {
    const segmentation = await net.segmentPerson(source, collectSegmentationOptions())
    const flipHorizontal = false
    const options = collectBokehOptions()
    bodyPix.drawBokehEffect(destination, source, segmentation, options.backgroundBlurAmount, options.edgeBlurAmount, flipHorizontal)
  }

  const imageFrame = async (source, destination) => {

    const offscreen = prepareOffscreen('offscreen', destination)
    const g = offscreen.getContext('2d')
    g.save()

    // 元の絵を先に描画（セグメンテーションとの時差防止）
    g.drawImage(source, 0, 0)

    const segmentation = await net.segmentPerson(source, collectSegmentationOptions())
    const flipHorizontal = false

    const options = collectImageOptions()
    const mask = bodyPix.toMask(segmentation, {r: 0, g: 0, b: 0, a: 255}, {r: 0, g: 0, b: 0, a: 0})

    const width = destination.width
    const height = destination.height

    // create blur mask

    // mask を canvas に展開
    const maskCanvas = prepareOffscreen('mask', destination)
    maskCanvas.getContext('2d').putImageData(mask, 0, 0)

    // mask の境界をぼかして描画
    const clipCanvas = blur('blur', maskCanvas, options.edgeBlurAmount)

    // 人物周辺以外を落とし
    g.globalCompositeOperation = 'destination-in'
    g.drawImage(clipCanvas, 0, 0)

    // 背景をぼかして描画
    try {
      g.globalCompositeOperation = 'destination-over'
      g.filter = `blur(${options.backgroundBlurAmount}px)`
      const image = getBackgroundImage()
      if (image?.complete) {
        g.drawImage(image, 0, 0, width, height)
      } else {
        g.fillStyle = imageBackgroundColor.value
        g.fillRect(0, 0, width, height)
      }
    } catch {
      // CORS ?
      g.fillStyle = imageBackgroundColor.value
      g.fillRect(0, 0, width, height)
    }

    g.restore()

    // 転送
    destination.getContext('2d').drawImage(offscreen, 0, 0)

  }

  const effectFrame = async (source, destination) => {

    if (effectOff.checked) {
      await copyFrame(source, destination)
      return
    }

    if (effectBokeh.checked) {
      await bokehFrame(source, destination)
      return
    }

    if (effectMask.checked) {
      await imageFrame(source, destination)
      return
    }

    // ?
    await copyFrame(source, destination)

  }

  const createTrackProcessor = () => {

    const panel = document.createElement('div')
    panel.className = 'processor'
    panel.style.position = 'fixed'
    panel.style.left = '48px'
    panel.style.bottom = '16px'
    panel.innerHTML = `
      <div style="display: none">
        <canvas class="processor-input"></canvas>
        <canvas class="processor-output"></canvas>
      </div>
    `
    document.body.append(panel)

    return panel

  }

  let activeTrack = null

  const effectTrack = input => {

    if (activeTrack) {
      activeTrack.stop()
      activeTrack = null
    }

    console.info('bodypix-injector: setup track: %o', input)

    const panel = createTrackProcessor()

    const framerate = input.getSettings().frameRate
    const width = input.getSettings().width
    const height = input.getSettings().height
    const capture = new ImageCapture(input)

    const inputCanvas = panel.querySelector('.processor-input')
    inputCanvas.width = width
    inputCanvas.height = height

    const outputCanvas = panel.querySelector('.processor-output')
    outputCanvas.width = width
    outputCanvas.height = height

    const inputContext = inputCanvas.getContext('bitmaprenderer') // 2d

    const stream = outputCanvas.captureStream() // auto framerate
    const output = stream.getVideoTracks()[0]

    let timer = null

    const dispose = () => {
      timer && clearInterval(timer)
      panel.remove()
    }

    let running = false

    const processFrame = async () => {
      if (input.readyState !== 'live') {
        dispose()
        return
      }
      if (output.readyState !== 'live') {
        dispose()
        return
      }
      if (input.muted) {
        return
      }

      if (running) {
        console.info('bodypix-injector: Waiting for previous frame completion, skipped.')
        return
      }

      const started = Date.now()
      try {

        running = true

        const bitmap = await capture.grabFrame()
        inputContext.transferFromImageBitmap(bitmap)
        await effectFrame(inputCanvas, outputCanvas)

        running = false

        console.debug('bodypix-injector: process frame in %o ms', Date.now() - started)
      } catch (e) {
        console.info('bodypix-injector: grabFrame rejected %o', e)
      }
    }

    const resume = () => {
      timer && clearInterval(timer)
      timer = setInterval(processFrame, 1000 / framerate)
    }

    const pause = () => {
      timer && clearInterval(timer)
      timer = null
    }

    input.addEventListener('ended', e => {
      console.warn('bodypix-injector: Track ended')
      dispose()
    })
    input.addEventListener('mute', e => {
      console.warn('bodypix-injector: Track muted')
      pause()
    })
    input.addEventListener('unmute', e => {
      console.warn('bodypix-injector: Track unmuted')
      resume()
    })

    output.addEventListener('ended', e => {
      console.warn('bodypix-injector: Output track ended')
      dispose()
    })

    resume()

    activeTrack = output

    return output
  }

  const effectStream = input => {
    // audio -> as-is
    // video -> effect
    const output = new MediaStream()
    input.getVideoTracks().forEach(t => output.addTrack(effectTrack(t)))
    input.getAudioTracks().forEach(t => output.addTrack(t))
    return output
  }

  const injectGetUserMedia = () => {

    const gum = navigator.mediaDevices.getUserMedia
    navigator.mediaDevices.getUserMedia = async constraints => {

      console.info('bodypix-injector: navigator.mediaDevices.getUserMedia called with constraint %o.', constraints)

      const video = constraints?.video
      if (!video || video === false) {
        // no video requested, do default
        return await gum.call(navigator.mediaDevices, constraints)
      }

      // default camera changed
      if (!video.deviceId && defaultCamera.value) {
        console.info('bodypix-injector: Video device not specified, Set deviceId constraint to %o', defaultCamera.value)
        if (video === true) {
          constraints = {video: {deviceId: defaultCamera.value}}
        } else {
          video.deviceId = defaultCamera.value
        }
      }

      const stream = await gum.call(navigator.mediaDevices, constraints)
      return effectStream(stream)

    }

  }

  injectGetUserMedia()

  // panel.querySelector('#test').addEventListener('click', async e => {
  //
  //   const stream = await navigator.mediaDevices.getUserMedia({
  //     audio: true,
  //     video: true,
  //   })
  //
  // })

})();

how-to-use.txt

script 要素でホストする場合

  <script defer src="body-pix-injector.js"></script>

あらゆるサイトでテストしてみる場合 Developer Console でこの Gist を即時実行

  fetch('https://gist.githubusercontent.com/iso2022jp/6da2133a96a90a06801f27b660292542/raw/fc2f8636e694b4e9b079641f5409b7c1f5f467c8/body-pix-injector.js').then(r => r.text()).then(eval)