noVNC display.js changed to use createImageBitmap for the img and blit operations. This will speedup image drawing to the canvas significantly

display.js

/*
 * noVNC: HTML5 VNC client
 * Copyright (C) 2019 The noVNC Authors
 * Licensed under MPL 2.0 (see LICENSE.txt)
 *
 * See README.md for usage and integration instructions.
 *
 * Modified by Mr-Shibari (Peter Hebels) to use createImageBitmap for the img and blit operations,
 * which will speedup image drawing to the canvas significantly. Also put the final drawimage (triggerd by flip operation)
 * in a promise and added hints to the canvas to disable alpha and antialias functions.
 *
 * Also added a simple FPS (flips per second) counter for testing the performance.
 *
 * If you want to test this you can simply replace the contents of the current version display.js file with this.
 *
 * In Firefox there seems to be a problem with createImageBitmap not releasing resources the right way, 
 * even after calling bitmap.close() on the generated bitmap. This will cause the GC to kick in and cause a
 * short jank which can be annoying. Not found a solution for this and there is a possiblility that it's a 
 * bug in the browser itself.
 */

import * as Log from './util/logging.js';
import { toSigned32bit } from './util/int.js';

export default class Display {
    constructor(target) {
        this._drawCtx = null;

        this._renderQ = [];  // queue drawing actions for in-oder rendering
        this._flushPromise = null;

        // the full frame buffer (logical canvas) size
        this._fbWidth = 0;
        this._fbHeight = 0;

        this._prevDrawStyle = "";

        Log.Debug(">> Display.constructor");

        // The visible canvas
        this._target = target;

        if (!this._target) {
            throw new Error("Target must be set");
        }

        if (typeof this._target === 'string') {
            throw new Error('target must be a DOM element');
        }

        if (!this._target.getContext) {
            throw new Error("no getContext method");
        }

        this._viewportLoc = { 'x': 0, 'y': 0, 'w': this._target.width, 'h': this._target.height };

        // the visible canvas viewport (i.e. what actually gets seen)
        this._targetCtx = this._target.getContext('2d', { 
            alpha: false, 
            antialias: false
        });

        // The hidden canvas, where we do the actual rendering
        this._backbuffer = new OffscreenCanvas(this._target.width, this._target.height); //document.createElement('canvas');
        this._drawCtx = this._backbuffer.getContext('2d', { 
            alpha: false, 
            antialias: false
        });

        this._damageBounds = { left: 0, top: 0,
                               right: this._backbuffer.width,
                               bottom: this._backbuffer.height };

        Log.Debug("User Agent: " + navigator.userAgent);

        Log.Debug("<< Display.constructor");

        // ===== PROPERTIES =====

        this._scale = 1.0;
        this._clipViewport = false;
        this._frame_drawn = true;
        this._bitmap_ready = true;

        this._targetCtx.imageSmoothingEnabled = false;
        this._drawCtx.imageSmoothingEnabled = false;

        this.times = [];
        this.fps = 0;
    }

    // ===== PROPERTIES =====

    get scale() { return this._scale; }
    set scale(scale) {
        this._rescale(scale);
    }

    get clipViewport() { return this._clipViewport; }
    set clipViewport(viewport) {
        this._clipViewport = viewport;
        // May need to readjust the viewport dimensions
        const vp = this._viewportLoc;
        this.viewportChangeSize(vp.w, vp.h);
        this.viewportChangePos(0, 0);
    }

    get width() {
        return this._fbWidth;
    }

    get height() {
        return this._fbHeight;
    }

    // ===== PUBLIC METHODS =====

    viewportChangePos(deltaX, deltaY) {
        const vp = this._viewportLoc;
        deltaX = Math.floor(deltaX);
        deltaY = Math.floor(deltaY);

        if (!this._clipViewport) {
            deltaX = -vp.w;  // clamped later of out of bounds
            deltaY = -vp.h;
        }

        const vx2 = vp.x + vp.w - 1;
        const vy2 = vp.y + vp.h - 1;

        // Position change

        if (deltaX < 0 && vp.x + deltaX < 0) {
            deltaX = -vp.x;
        }
        if (vx2 + deltaX >= this._fbWidth) {
            deltaX -= vx2 + deltaX - this._fbWidth + 1;
        }

        if (vp.y + deltaY < 0) {
            deltaY = -vp.y;
        }
        if (vy2 + deltaY >= this._fbHeight) {
            deltaY -= (vy2 + deltaY - this._fbHeight + 1);
        }

        if (deltaX === 0 && deltaY === 0) {
            return;
        }
        Log.Debug("viewportChange deltaX: " + deltaX + ", deltaY: " + deltaY);

        vp.x += deltaX;
        vp.y += deltaY;

        this._damage(vp.x, vp.y, vp.w, vp.h);

        this.flip();
    }

    viewportChangeSize(width, height) {

        if (!this._clipViewport ||
            typeof(width) === "undefined" ||
            typeof(height) === "undefined") {

            Log.Debug("Setting viewport to full display region");
            width = this._fbWidth;
            height = this._fbHeight;
        }

        width = Math.floor(width);
        height = Math.floor(height);

        if (width > this._fbWidth) {
            width = this._fbWidth;
        }
        if (height > this._fbHeight) {
            height = this._fbHeight;
        }

        const vp = this._viewportLoc;
        if (vp.w !== width || vp.h !== height) {
            vp.w = width;
            vp.h = height;

            const canvas = this._target;
            canvas.width = width;
            canvas.height = height;

            // The position might need to be updated if we've grown
            this.viewportChangePos(0, 0);

            this._damage(vp.x, vp.y, vp.w, vp.h);
            this.flip();

            // Update the visible size of the target canvas
            this._rescale(this._scale);

            this._targetCtx.imageSmoothingEnabled = false;
            this._drawCtx.imageSmoothingEnabled = false;
        }
    }

    absX(x) {
        if (this._scale === 0) {
            return 0;
        }
        return toSigned32bit(x / this._scale + this._viewportLoc.x);
    }

    absY(y) {
        if (this._scale === 0) {
            return 0;
        }
        return toSigned32bit(y / this._scale + this._viewportLoc.y);
    }

    resize(width, height) {
        this._prevDrawStyle = "";

        this._fbWidth = width;
        this._fbHeight = height;

        const canvas = this._backbuffer;
        if (canvas.width !== width || canvas.height !== height) {

            // We have to save the canvas data since changing the size will clear it
            let saveImg = null;
            if (canvas.width > 0 && canvas.height > 0) {
                saveImg = this._drawCtx.getImageData(0, 0, canvas.width, canvas.height);
            }

            if (canvas.width !== width) {
                canvas.width = width;
            }
            if (canvas.height !== height) {
                canvas.height = height;
            }

            if (saveImg) {
                this._drawCtx.putImageData(saveImg, 0, 0);
            }
        }

        // Readjust the viewport as it may be incorrectly sized
        // and positioned
        const vp = this._viewportLoc;
        this.viewportChangeSize(vp.w, vp.h);
        this.viewportChangePos(0, 0);
    }

    getImageData() {
        return this._drawCtx.getImageData(0, 0, this.width, this.height);
    }

    toDataURL(type, encoderOptions) {
        return this._backbuffer.toDataURL(type, encoderOptions);
    }

    toBlob(callback, type, quality) {
        return this._backbuffer.toBlob(callback, type, quality);
    }

    // Track what parts of the visible canvas that need updating
    _damage(x, y, w, h) {
        if (x < this._damageBounds.left) {
            this._damageBounds.left = x;
        }
        if (y < this._damageBounds.top) {
            this._damageBounds.top = y;
        }
        if ((x + w) > this._damageBounds.right) {
            this._damageBounds.right = x + w;
        }
        if ((y + h) > this._damageBounds.bottom) {
            this._damageBounds.bottom = y + h;
        }
    }

    // Update the visible canvas with the contents of the
    // rendering canvas
    flip(fromQueue) {
        if (this._renderQ.length !== 0 && !fromQueue) {
            this._renderQPush({
                'type': 'flip'
            });
        } else {
            let x = this._damageBounds.left;
            let y = this._damageBounds.top;
            let w = this._damageBounds.right - x;
            let h = this._damageBounds.bottom - y;

            let vx = x - this._viewportLoc.x;
            let vy = y - this._viewportLoc.y;

            if (vx < 0) {
                w += vx;
                x -= vx;
                vx = 0;
            }
            if (vy < 0) {
                h += vy;
                y -= vy;
                vy = 0;
            }

            if ((vx + w) > this._viewportLoc.w) {
                w = this._viewportLoc.w - vx;
            }
            if ((vy + h) > this._viewportLoc.h) {
                h = this._viewportLoc.h - vy;
            }

            if((w > 0) && (h > 0)) {
                if(this._frame_drawn == true) {
                    this._frame_drawn = false;

                    const drawPromise = new Promise((resolve) => {
                        this._targetCtx.drawImage(this._backbuffer, x, y, w, h, vx, vy, w, h);

                        const now = performance.now();
                        while (this.times.length > 0 && this.times[0] <= now - 1000) {
                           this.times.shift();
                        }
                        this.times.push(now);
                        this.fps = this.times.length;                     

                        this._targetCtx.clearRect(10, 30, 100, 40);
                        this._targetCtx.font = "30px Arial";
                        this._targetCtx.fillStyle = "red";
                        this._targetCtx.fillText(this.fps+" fps", 10, 60);

                        this._frame_drawn = true;

                        this._damageBounds.left = this._damageBounds.top = 65535;
                        this._damageBounds.right = this._damageBounds.bottom = 0;

                        resolve(true);
                    });

                    drawPromise.then();
                }
            }
        }
    }

    pending() {
        return this._renderQ.length > 0;
    }

    flush() {
        if (this._renderQ.length === 0) {
            return Promise.resolve();
        } else {
            if (this._flushPromise === null) {
                this._flushPromise = new Promise((resolve) => {
                    this._flushResolve = resolve;
                });
            }
            return this._flushPromise;
        }
    }

    fillRect(x, y, width, height, color, fromQueue) {
        if (this._renderQ.length !== 0 && !fromQueue) {
            this._renderQPush({
                'type': 'fill',
                'x': x,
                'y': y,
                'width': width,
                'height': height,
                'color': color
            });
        } else {
            this._setFillColor(color);
            this._drawCtx.fillRect(x, y, width, height);
            this._damage(x, y, width, height);
        }
    }

    copyImage(oldX, oldY, newX, newY, w, h, fromQueue) {
        if (this._renderQ.length !== 0 && !fromQueue) {
            this._renderQPush({
                'type': 'copy',
                'oldX': oldX,
                'oldY': oldY,
                'x': newX,
                'y': newY,
                'width': w,
                'height': h,
            });
        } else {
            this._drawCtx.drawImage(this._backbuffer,
                                    oldX, oldY, w, h,
                                    newX, newY, w, h);

            this._damage(newX, newY, w, h);
        }
    }

    imageRect(x, y, width, height, mime, arr) {
        /* The internal logic cannot handle empty images, so bail early */
        if ((width === 0) || (height === 0)) {
            return;
        }

        this._renderQPush({
            'type': 'img',
            'img': new Blob([arr], { type: mime }),
            'x': x,
            'y': y
        });
    }

    blitImage(x, y, width, height, arr, offset, fromQueue) {
        // NB(directxman12): it's technically more performant here to use preallocated arrays,
        // but it's a lot of extra work for not a lot of payoff -- if we're using the render queue,
        // this probably isn't getting called *nearly* as much
        let newArr = new Uint8Array(width * height * 4);
        newArr.set(new Uint8Array(arr.buffer, 0, newArr.length));

        let data = new Uint8ClampedArray(
            newArr.buffer,
            newArr.byteOffset + newArr.offset,
            width * height * 4
        );

        let img = new ImageData(data, width, height);

        this._renderQPush({
            'type': 'img',
            'img': img,
            'x': x,
            'y': y
        });

        data = null;
        newArr = null;
        img = null;
    }

    drawImage(img, x, y) {
        this._drawCtx.drawImage(img, x, y);
        this._damage(x, y, img.width, img.height);
    }

    autoscale(containerWidth, containerHeight) {
        let scaleRatio;

        if (containerWidth === 0 || containerHeight === 0) {
            scaleRatio = 0;

        } else {

            const vp = this._viewportLoc;
            const targetAspectRatio = containerWidth / containerHeight;
            const fbAspectRatio = vp.w / vp.h;

            if (fbAspectRatio >= targetAspectRatio) {
                scaleRatio = containerWidth / vp.w;
            } else {
                scaleRatio = containerHeight / vp.h;
            }
        }

        this._rescale(scaleRatio);
    }

    // ===== PRIVATE METHODS =====

    _rescale(factor) {
        this._scale = factor;
        const vp = this._viewportLoc;

        // NB(directxman12): If you set the width directly, or set the
        //                   style width to a number, the canvas is cleared.
        //                   However, if you set the style width to a string
        //                   ('NNNpx'), the canvas is scaled without clearing.
        const width = factor * vp.w + 'px';
        const height = factor * vp.h + 'px';

        if ((this._target.style.width !== width) ||
            (this._target.style.height !== height)) {
            this._target.style.width = width;
            this._target.style.height = height;
        }
    }

    _setFillColor(color) {
        const newStyle = 'rgb(' + color[0] + ',' + color[1] + ',' + color[2] + ')';
        if (newStyle !== this._prevDrawStyle) {
            this._drawCtx.fillStyle = newStyle;
            this._prevDrawStyle = newStyle;
        }
    }

    _renderQPush(action) {
        this._renderQ.push(action);
        if (this._renderQ.length === 1) {
            // If this can be rendered immediately it will be, otherwise
            // the scanner will wait for the relevant event
            this._scanRenderQ();
        }
    }

    _resumeRenderQ() {
        // "this" is the object that is ready, not the
        // display object
        this.removeEventListener('load', this._noVNCDisplay._resumeRenderQ);
        this._noVNCDisplay._scanRenderQ();
    }

    _scanRenderQ() {
        let ready = true;

        while (ready && this._renderQ.length > 0) {
            let a = this._renderQ[0];
            switch (a.type) {
                case 'flip':
                    this.flip(true);
                    break;
                case 'copy':
                    this.copyImage(a.oldX, a.oldY, a.x, a.y, a.width, a.height, true);
                    break;
                case 'fill':
                    this.fillRect(a.x, a.y, a.width, a.height, a.color, true);
                    break;
                case 'img':
                    if(this._bitmap_ready) {
                        this._bitmap_ready = false;

                        createImageBitmap(a.img, {premultiplyAlpha: "none", colorSpaceConversion: "none"}).then((bitmap_data) => {
                            this.drawImage(bitmap_data, a.x, a.y);
                            this._bitmap_ready = true;

                            bitmap_data.close();
                            bitmap_data = null;
                            delete a.img;

                            delete this._renderQ[0];
                            this._renderQ.shift();

                            this._scanRenderQ();
                        });
                    }

                    ready = false;
                    break;
            }

            if (ready) {
                delete this._renderQ[0];
                this._renderQ.shift();
            }
        }

        if (this._renderQ.length === 0 &&
            this._flushPromise !== null) {
            this._flushResolve();
            this._flushPromise = null;
            this._flushResolve = null;
        }
    }
}