A Three.js / Socket.io / Node.js render and transmit test.

a-render-and-transmit-test.md

This example renders thirty frames of the Three.js 'your first scene' example and sends them to a Node.js microservice, which saves them to the filesystem.

test-render-client.html

• Creates the scene, camera, renderer, and kicks off the render loop, which stops after 30 frames have been rendered.
• As an optimization, it doesn't add the Three.js canvas to the browser DOM, rendering it offscreen, but reporting progress.
• It extracts the data for each frame using canvas.toDataURL(), sending that to a web worker process for transmission.
• When all frames are rendered, it sends a 'done' message to the worker.

test-render-worker.js

• Sets up a queue for incoming frames to be sent to the server.
• Services the queue at an interval.
  • If sending is in progress, the service method exits, doing nothing.
  • If rendering is done and the queue is empty, it closes the socket and terminates the worker.
  • Otherwise, if the queue has items, it sets the sending flag, emits a frame event on the socket, with a callback.
    • The callback clears the sending flag.
• The onmessage handler checks the message type.
  • If 'done' it sets the done flag.
  • If 'frame', it pushes the frame onto the queue.

test-render-server.js

• Creates a Socket.io server, setting listeners for 'frame' and 'disconnect' events.
• The onFrame handler takes data and a callback.
  • It writes the data to disk as '/var/tmp/test-render-server//frame-xxxx.png'
  • Calls the callback, sending the test-render-worker confirmation that it got the data.
• The onDisconnect handler removes the event listeners from the connection object.

NOTES: Typical output (without the browser's debugger being open):

Total Frames: 30 Total time: 4165ms ms per frame: 138.83333333333334

This output on the web page only indicates how long it took to render the frames and feed them to the web worker, not the total time to render AND transmit. We're not concerned about transmission time, only how long it takes to render a frame.

This approach takes about 4 seconds to render 1 second's worth of frames. And it's only rotating a simple cube. A more complex scene would of course take longer.

This setup would be fine if you're not trying to sync scene elements to an audio track, but in my real application, I am. Audio is playing, and spectrum analysis is being done continuously so that objects in the scene may have properties such as position and size be computed from the audio's low, mid, high, or overall volume. What is likely to happen here is that when the server tries to make a video from these frames and the audio, they will be out of sync.

If you comment out line 65 of test-render-client.html (which calls canvas.toDataURL()), the time to render a frame drops to around 20ms or less. Therefore the process of extracting the data cannot happen inside the rendering loop.

test-render-client.html

<!-- test-render-client.html -->
<script src="https://rawgithub.com/mrdoob/three.js/master/build/three.js"></script>
<script>

    // Create scene and renderer, connect to canvas
    var scene = new THREE.Scene();
    var camera = new THREE.PerspectiveCamera(75, 1920/1080, 0.1, 1000);
    var geometry = new THREE.CubeGeometry(1,1,1);
    var material = new THREE.MeshBasicMaterial({color: 0x00ff00});
    var cube = new THREE.Mesh(geometry, material);
    var renderer = new THREE.WebGLRenderer();
    scene.add(cube);
    camera.position.z = 5;
    renderer.setSize(1920,1080); // HD / 1080p
    
    // Progress div
    var outputDiv = document.createElement("div");
    document.body.appendChild(outputDiv);
    
    // Add renderer's canvas to the DOM (it's faster not to, though)
    var canvas = renderer.domElement;
    // document.body.appendChild(canvas); // adds ~30ms / frame
    
    // Create a web worker to transmit the frames on another thread
    var worker;
    if (typeof(Worker) !== 'undefined') {
        worker = new Worker('test-render-worker.js');
    } else {
        throw new Error('No Web Worker support');
    }

    // Create a predetermined number of frames then disconnect
    var message = null;
    var ordinal = 0;
    var cutoff = 30;
    var done = false;
    var endTime = null;
    var startTime = null;
    var totalTime = null;
    var frameRenderTime = null;

    var render = function () {

        if (!done) {

            if (ordinal === cutoff) {
                // Notify the worker that we're done
                endTime = Date.now();
                done = true;
                message = {type:'done'};
                worker.postMessage(message);

                // Report total frames and render time on page
                totalTime = (endTime-startTime);
                frameRenderTime = totalTime/ordinal;
                outputDiv.innerHTML =
                        "Total Frames: " + ordinal +
                        "<br/>Total time: " + totalTime + "ms" +
                        "<br/>ms per frame: " + frameRenderTime;
            } else {
                // Send the rendered frame to the web worker
                message = {
                    type: 'frame',
                    ordinal: ordinal++,
                    data: canvas.toDataURL('image/png') // ~116 ms!!!
                };
                worker.postMessage(message); // ~2ms

                // Kick off the next frame render
                requestAnimationFrame(render);
                renderer.render(scene, camera); // ~ 20ms

                cube.rotation.x += 0.1;
                cube.rotation.y += 0.1;

                outputDiv.innerHTML = "Rendering frame "+ordinal; // ~4ms
            }
        }
    };

    // One, two, three, go!
    startTime = Date.now();
    render();
</script>

test-render-server.js

// test-render-server.js

// Required modules
var fs = require('fs');
var mkdirp = require('mkdirp');

// Create the socket server
const PORT = 3000;
var socket = require('socket.io')(PORT);
console.log('Socket server listening on port: '+PORT);

// Handle connections
socket.on('connection', function(client) {

    // Listen for frame and disconnect events
    client.on('frame', onFrame);
    client.on('disconnect', onDisconnect);

    // Create output folder for this client
    var output = "/var/tmp/test-render-server/" + client.id + "/";
    mkdirp(output);

    // Handle a frame event from the client
    function onFrame(frame, callback) {
        console.log('Received frame: "' + frame.ordinal + '" from client: ' + client.id);
        // Assemble filename
        var zeroPadFrame = ("000" + frame.ordinal).slice(-3);
        var filename = output + "frame-"+zeroPadFrame+".png";

        // Drop 'data:/base64png,' header
        frame.data = frame.data.split(',')[1];

        // Create the file
        var file = new Buffer(frame.data, 'base64');
        fs.writeFile(filename, file.toString('binary'), 'binary');

        // Acknowledge receipt
        callback();
    }

    // Handle a disconnection from the client
    function onDisconnect() {
        console.log('Received: disconnect event from client: ' + client.id);
        client.removeListener('frame', onFrame);
        client.removeListener('disconnect', onDisconnect);
    }

});

test-render-worker.js

// test-render-worker.js

// Connect to the socket server
self.importScripts('https://cdn.socket.io/socket.io-1.4.5.js');
var socket = io.connect('http://localhost:3000');

// Queue the images to be transmitted,
// servicing the queue by timer, and
// closing the socket and worker when
// the last image has been sent.
var frame, queue = [], done = false, sending=false,
    timer = setInterval(serviceQueue,30);

function serviceQueue(){
    if (sending) return;
    if (queue.length > 0) {
        sending=true;
        frame = queue.shift();
        socket.emit('frame', frame, function(){
            console.log('[WORKER]: Send complete '+ frame.ordinal);
            sending=false;
        });
        console.log('[WORKER]: Sending frame '+ frame.ordinal);
    } else if (done && queue.length === 0) {
        clearInterval(timer);
        socket.close();
        close();
    }
}

// Handle messages from the web page
onmessage = function (e){
    var message = e.data;
    switch (message.type) {

        // Add a frame to the queue
        case 'frame':
            delete message['type'];
            console.log('[WORKER]: Received frame '+ message.ordinal);
            queue.push(message);
            break;

        // That's all, folks
        case 'done':
            console.log('[WORKER]: Done. Closing socket and web worker.');
            done = true;
            break;
    }
};