Mocking a browser's MediaStreams API so you can play multiple audio files

Browser MediaStreams API Mocking.md

Mocking a browser's MediaStreams API

This script is used by Puppeteer tests to be able to mock a browser's MediaStreams API allowing you to play multiple audio files.

The script is necessary as the Chrome flag --use-file-for-fake-audio-capture will only allow one audio file to be chosen, and you have no control over when it will be played.

await webRtcFrame.evaluate(mockMediaDevicesScript);
webRtcFrame.evaluate(() => window.mockMediaDevice.playAudio(/* URL */));

Read my blog article Automated tests using Genesys Cloud's WebRTC softphone to see how I am using this.

Example usage

Bundle the TypeScript into a JS file

npx esbuild ./index.ts --bundle --outfile=mocked_media_stream.bundle.js --platform=browser --target=es6 --minify

If you don't want to build it yourself then I've done it for you into the mocked_media_stream.bundle.js listed in this Gist.

Run the HTTP server that will be used to expose the audio files

This solution replaces the MediaStreams within a particular page. This means we will need to pass the audio you want to play over this stream to that page.

Ideally our Puppeteer script would just pass the byte array of the audio into the page, but alas the array will be too big. So this is the workaround. By hosting the audio files locally we can save them to a file and then pass the URL to the page instead. The page will then download the file (see step 5 below) and pass the byte array to the mocked Media Devices playAudio function.

npx http-server audio-files -p 8080 --cors

Update your Puppeteer Script

// 1. Read in the JS bundled above
const mockMediaDevicesScript = readFileSync("./dist/bundle.js", "utf8");

// 2. Get IFrame for WebRTC phone
const webRtcFrame = await browser.waitForTarget((page) =>
        page.url().includes("/crm/webrtc.html"),
      );

// 3. Run the script within the WebRTC frame to replace the MediaStream API
await webRtcFrame.evaluate(mockMediaDevicesScript);

// 4. Run a JavaScript function within the IFrame when you want to play an audio file against the MediaStream
webRtcFrame.evaluate((fileNameIn) => {
    window.mockMediaDevice.playAudio(fileNameIn);
}, `http://localhost:8080/${filename}`);

Note

The recommendation for overriding Web APIs is using the Page.evaluateOnNewDocument() method, however for some reason this doesn't work for navigator.mediaDevices. It is as if it is being overridden at a later time by the --use-fake-device-for-media-stream flag.

global.d.ts

interface Window {
  mockMediaDevice: {
    audioContext: AudioContext;
    source: AudioBufferSourceNode | null;
    destination: MediaStreamAudioDestinationNode | null;
    playAudio: (url: string) => void;
  };
}

index.ts

import { mockEnumerateDevices } from './mockEnumerateDevices';
import { mockGetUserMedia } from './mockGetUserMedia';

navigator.mediaDevices.enumerateDevices = mockEnumerateDevices();
navigator.mediaDevices.getUserMedia = mockGetUserMedia(navigator.mediaDevices.getUserMedia);
console.log('MockMediaDevices script ran');

mocked_media_stream.bundle.js

(()=>{var f=Object.defineProperty,v=Object.defineProperties;var M=Object.getOwnPropertyDescriptors;var c=Object.getOwnPropertySymbols;var D=Object.prototype.hasOwnProperty,l=Object.prototype.propertyIsEnumerable;var d=(o,e,i)=>e in o?f(o,e,{enumerable:!0,configurable:!0,writable:!0,value:i}):o[e]=i,a=(o,e)=>{for(var i in e||(e={}))D.call(e,i)&&d(o,i,e[i]);if(c)for(var i of c(e))l.call(e,i)&&d(o,i,e[i]);return o},t=(o,e)=>v(o,M(e));var n={deviceId:"ccadbbdd8b64da6681ad4fdd7c1191fd64cb64778b74cc4623e5e2bd3adf5cfe",kind:"audioinput",label:"Fake Microphone",groupId:"129c74c2662b7d877ab4a236d633f05e285d1444c93775390d863b1a1e11d75e"},r={deviceId:"b29700fa8cf469ac3716a00c4655268cf004ac6ca8ae71121a0cd9671d571aef",kind:"audiooutput",label:"Fake Speakers",groupId:"129c74c2662b7d877ab4a236d633f05e285d1444c93775390d863b1a1e11d75e"};function s(){return console.debug("MockMediaDevices - mockEnumerateDevices called"),()=>Promise.resolve([t(a({},n),{toJSON:()=>JSON.stringify(n)}),t(a({},r),{toJSON:()=>JSON.stringify(r)})])}function u(o){return window.mockMediaDevice={audioContext:new AudioContext,destination:null,source:null,playAudio(e){console.debug("MockMediaDevices - playAudio called"),this.audioContext.resume().then(()=>{fetch(e).then(i=>i.arrayBuffer()).then(i=>this.audioContext.decodeAudioData(i)).then(i=>{this.source&&this.source.disconnect(),this.source=this.audioContext.createBufferSource(),this.source.buffer=i,this.source.loop=!1,this.destination||(this.destination=this.audioContext.createMediaStreamDestination()),this.source.connect(this.destination),this.source.start(0)}).catch(i=>console.error("MockMediaDevices: Error playing audio:",i))}).catch(i=>{console.error("MockMediaDevices: Error resuming AudioContext:",i)})}},function(e){return console.debug("MockMediaDevices - getUserMedia called"),e&&e.audio&&(typeof e.audio=="boolean"&&e.audio||typeof e.audio=="object")?new Promise(m=>{window.mockMediaDevice.destination=window.mockMediaDevice.audioContext.createMediaStreamDestination(),window.mockMediaDevice.source&&window.mockMediaDevice.source.connect(window.mockMediaDevice.destination),m(window.mockMediaDevice.destination.stream)}):o(e)}}navigator.mediaDevices.enumerateDevices=s();navigator.mediaDevices.getUserMedia=u(navigator.mediaDevices.getUserMedia);console.log("MockMediaDevices script ran");})();

mockEnumerateDevices.ts

const microphone: Omit<MediaDeviceInfo, 'toJSON'> = {
  deviceId: 'ccadbbdd8b64da6681ad4fdd7c1191fd64cb64778b74cc4623e5e2bd3adf5cfe', // Randomly generated
  kind: 'audioinput',
  label: 'Fake Microphone',
  groupId: '129c74c2662b7d877ab4a236d633f05e285d1444c93775390d863b1a1e11d75e', // Randomly generated, same as Group ID below
};

const speaker: Omit<MediaDeviceInfo, 'toJSON'> = {
  deviceId: 'b29700fa8cf469ac3716a00c4655268cf004ac6ca8ae71121a0cd9671d571aef', // Randomly generated
  kind: 'audiooutput',
  label: 'Fake Speakers',
  groupId: '129c74c2662b7d877ab4a236d633f05e285d1444c93775390d863b1a1e11d75e',
};

export function mockEnumerateDevices(): typeof navigator.mediaDevices.enumerateDevices {
  console.debug('MockMediaDevices - mockEnumerateDevices called');
  return () =>
    Promise.resolve([
      {
        ...microphone,
        toJSON: () => JSON.stringify(microphone),
      },
      {
        ...speaker,
        toJSON: () => JSON.stringify(speaker),
      },
    ]);
}

mockGetUserMedia.ts

type GetUserMediaFunc = typeof navigator.mediaDevices.getUserMedia;

export function mockGetUserMedia(originalGetUserMedia: GetUserMediaFunc): GetUserMediaFunc {
  window.mockMediaDevice = {
    audioContext: new AudioContext(),
    destination: null,
    source: null,
    playAudio(url: string) {
      console.debug('MockMediaDevices - playAudio called');
      this.audioContext
        .resume()
        .then(() => {
          fetch(url)
            .then((response) => response.arrayBuffer())
            .then((arrayBuffer) => this.audioContext.decodeAudioData(arrayBuffer))
            .then((audioBuffer) => {
              if (this.source) {
                this.source.disconnect();
              }
              this.source = this.audioContext.createBufferSource();
              this.source.buffer = audioBuffer;
              this.source.loop = false;
              if (!this.destination) {
                this.destination = this.audioContext.createMediaStreamDestination();
              }
              this.source.connect(this.destination);
              this.source.start(0);
            })
            .catch((error) => console.error('MockMediaDevices: Error playing audio:', error));
        })
        .catch((error) => {
          console.error('MockMediaDevices: Error resuming AudioContext:', error);
        });
    },
  };

  return function(constraints) {
    console.debug('MockMediaDevices - getUserMedia called');
    const audioDeviceRequested =
      constraints &&
      constraints.audio &&
      ((typeof constraints.audio === 'boolean' && constraints.audio) ||
        typeof constraints.audio === 'object');

    if (!audioDeviceRequested) {
      return originalGetUserMedia(constraints);
    }

    return new Promise((resolve) => {
      window.mockMediaDevice.destination =
        window.mockMediaDevice.audioContext.createMediaStreamDestination();

      if (window.mockMediaDevice.source) {
        window.mockMediaDevice.source.connect(window.mockMediaDevice.destination);
      }
      resolve(window.mockMediaDevice.destination.stream);
    });
  };
}