compileJS.js

'use strict';

module.exports = function compileJS(grunt) {
    const babel = require('babel-core'),
        Concat = require('concat-with-sourcemaps'),
        config = grunt.config.get('tt'),
        configHelper = require('../tt/config.js'),
        fileSeparator = ';\n',
        fs = require('fs'),
        path = config.get('dir.public') + '/js',
        uglify = require('uglify-js'),

        getSDKOrCache = memoize(getSDK),
        getVendorsOrCache = memoize(getVendors);

    grunt.registerTask('compileJS-apps', 'Create minified, transpiled JS bundles for each application', function() {
        const applications = configHelper.getApplications(),
              done = this.async();

        Promise.all(
            applications.map(generateAllMinJS)
        ).then(done);
    });

    grunt.registerTask('compileJS-serverSDK', 'Create SDK bundle for Node application', function() {
        const done = this.async();
        generateServerSDK().then(done);
    });

    /**
     * Get all.min.js bundle for an application, and write it to disk
     * @param application
     * @returns {Promise.<undefined|Error>}
     */
    function generateAllMinJS(application) {
        const start = new Date(),
            bundleName = `all.min.js`,
            parts = [
                getVendorsOrCache(),
                getSDKOrCache(),
                getModules(application),
                getTemplates(application)
            ];

        return Promise.all(parts).then(parts => {
            let bundle = concatResources(parts, bundleName);
            bundle.writeFile(`${path}/${application.name}`, true);
            successMessage();
        }).catch(
            errorGenerator(`${application.name}/${bundleName}`)
        );

        function successMessage() {
            let timing = new Date() - start;
            console.log(`Compiled ${application.name}/${bundleName} after ${timing / 1000} seconds`);
        }
    }

    /**
     * Create and write an SDK bundle, for use by the Node app
     */
    function generateServerSDK() {
        const start = new Date();

        return getSDKOrCache()
        .then(sdk => {
            sdk.writeFile(path, false);
            let timing = new Date() - start;
            console.log(`Compiled SDK bundle in ${timing / 1000} seconds`);
        });
    }

    /**
     * Get module file bundle for application
     * @param application
     * @returns {Promise.<Resource|Error>}
     */
    function getModules(application) {
        return getModuleResources()
            .then(concatenate)
            .then(transpile)
            .then(minify)
            .catch(errorGenerator(`${application.name}/modules.min.js`));

        function getModuleResources() {
            return getResources(configHelper.getModuleFiles(application));
        }

        function concatenate(resourceList) {
            return concatResources(resourceList, 'modules.es6');
        }

        function transpile(resource) {
            const bundleName = `${application.name}/modules.es5`;
            return transpileResource(resource, bundleName);
        }

        function minify(resource) {
            const bundleName = `modules.min.js`;
            return minifyResource(resource, bundleName);
        }
    }

    /**
     * Get Tictrac SDK bundle
     * @returns {Promise.<Resource|Error>}
     */
    function getSDK() {
        return getSDKResources()
            .then(concatenate)
            .then(transpile)
            .then(minify)
            .catch(errorGenerator('tictrac-sdk.min.js'));

        function getSDKResources() {
            return getResources(configHelper.getSDKFiles());
        }

        function concatenate(resourceList) {
            return concatResources(resourceList, 'tictrac-sdk.es6');
        }

        function transpile(resource) {
            const bundleName = `tictrac-sdk.es5`;
            return transpileResource(resource, bundleName);
        }

        function minify(resource) {
            const bundleName = `tictrac-sdk.min.js`;
            return minifyResource(resource, bundleName, {
                wrap: true,
                mangle: false
            });
        }
    }

    /**
     * Get the vendors.js bundle
     * This includes all third-party scripts
     * @returns {Promise.<Resource|Error>}
     */
    function getVendors() {
        return getModuleResources()
            .then(concatenate)
            .then(minify)
            .catch(errorGenerator(`vendors.js`));

        function getModuleResources() {
            return getResources(configHelper.getVendorFiles());
        }

        function concatenate(resourceList) {
            return concatResources(resourceList, 'vendors.js');
        }

        function minify(resource) {
            const bundleName = `vendors.min.js`;
            return minifyResource(resource, bundleName, {
                mangle: false,
                compress: false,
                preserveComments: false
            });
        }
    }

    /**
     * Get the templates.js file for the specified application
     * @param {String} application
     * @returns {Promise.<Resource|Error>}
     */
    function getTemplates(application) {
        const fileName = `${config.get('dir.public')}/js/${application.name}/templates.js`;

        return getResource(fileName)
            .then(minify);

        function minify(resource) {
            return minifyResource(resource, `${application.name}/template.js`);
        }
    }

    /**
     * Concatenate several Resources into a new Resource, with a sourcemap
     * @param {Array.<Resource>} resourceList
     * @param {String} bundleName
     * @returns {Resource}
     */
    function concatResources(resourceList, bundleName) {
        const concatenator = new Concat(true, bundleName, fileSeparator);

        resourceList.forEach(resource => {
            let resourceLocation = resource.exists ? resource.getBuildLocation() : resource.filename;
            concatenator.add(resourceLocation, resource.content, resource.sourceMap);
        });

        return new Resource({
            filename: bundleName,
            sourceMap: concatenator.sourceMap,
            content: concatenator.content,
            exists: false
        });
    }

    /**
     * Transpile and ES6 resource into an ES5 resource
     * @param {Resource} resource
     * @param {String} bundleName
     * @returns {Resource}
     */
    function transpileResource(resource, bundleName) {
        const transformation = babel.transform(resource.content, {
            filename: bundleName,
            presets: ['es2015'],
            inputSourceMap: resource.sourceMap,
            sourceMap: true,
            compact: true
        });

        return new Resource({
            filename: bundleName,
            sourceMap: transformation.map,
            content: transformation.code,
            exists: false
        });
    }

    /**
     * Transform a resource into a minified resource, in memory
     * @param {Resource} inputResource
     * @param {String} outputFilename
     * @param {Object=} minifierOpts - see Uglify options
     * @returns {Resource}
     */
    function minifyResource(inputResource, outputFilename, minifierOpts) {
        minifierOpts = minifierOpts || {};
        minifierOpts.fromString = true;

        // If we're minifying with a sourcemap, wire up source map transformations with uglify
        if (inputResource.sourceMap) {
            minifierOpts.inSourceMap = inputResource.sourceMap;
            minifierOpts.outSourceMap = outputFilename + '.map';
        }

        const minified = uglify.minify(inputResource.content, minifierOpts);

        return new Resource({
            filename: outputFilename,
            sourceMap: minified.map,
            content: minified.code,
            exists: false
        });
    }

    /**
     * Create an error handler for a particular bundle job
     * @param {String} bundleName
     * @returns {Function}
     */
    function errorGenerator(bundleName) {
        return function(e) {
            console.log(`Error generating ${bundleName}: ${e}, at ${e.stack}`);
        };
    }

    /**
     * Maps an Array.<String> into getResource
     */
    function getResources(fileNames) {
        return Promise.all(fileNames.map(getResource));
    }

    /**
     * Load a file into a Resource
     * @param {String} filename
     * @returns {Promise.<Resource|Error>}
     */
    function getResource(filename) {
        return new Promise((resolve, reject)=> {
            fs.readFile(filename, 'utf-8', (error, data)=> {
                if (error) {
                    reject(error);
                } else {
                    resolve(data);
                }
            });
        }).then(content => {
            return new Resource({content, filename, exists: true});
        });
    }

    /**
     * Like a file, but can be 'virtualised' (i.e. only exists in memory),
     * and carries metadata like sourcemaps.
     * @param opts
     * @constructor
     */
    function Resource(opts) {
        this.content = opts.content.toString();
        this.sourceMap = opts.sourceMap ? toObject(opts.sourceMap) : null;
        this.filename = opts.filename || '';
        this.exists = opts.exists;

        /**
         * Where will this file be copied to in the build directory?
         * This is necessary so that sourcemap users can browse the original files.
         * @returns {String}
         */
        this.getBuildLocation = function() {
            if (this.exists) {
                return toPublicFilename(this.filename);
            } else {
                throw new Error(
                    `Wanted publicly accessible location of ${this.filename}, ` +
                    `but file unwritten to disk and will be missed by grunt-copy:build`
                );
            }
        };

        /**
         * Write out file and any associated sourcemap
         * @param {String} filePath
         * @param {Boolean} withSourceMap - true
         */
        this.writeFile = function(filePath, withSourceMap) {
            withSourceMap = defaultArg(withSourceMap, true);
            const fileLocation = `${filePath}/${this.filename}`,
                mapFilename = `${this.filename}.map`,
                mapLocation = `${filePath}/${mapFilename}`;

            if (this.sourceMap && withSourceMap) {
                let sourceMapComment = `\n//# sourceMappingURL=${mapFilename}`;
                write(fileLocation, this.content + sourceMapComment);
                write(mapLocation, JSON.stringify(this.sourceMap));
            } else {
                write(fileLocation, this.content);
            }

            this.exists = true;

            function write(where, what) {
                grunt.file.write(where, what);
            }
        };
    }

    /**
     * Transform {String|Object} into {Object}
     */
    function toObject(value) {
        const type = typeof value;
        if (type === 'string') {
            return JSON.parse(value);
        } else {
            return value;
        }
    }

    /**
     * Translate a local filename into the eventual publicly-available location
     * @param {String} filename
     * @returns {String}
     */
    function toPublicFilename(filename) {
        if (filename.includes('node_modules')) {
            // /node_modules is mapped by Express under /vendor
            return filename.replace('node_modules', '/vendor');

        } else if (filename.includes('/public/')) {
            // Project files need to be given non-absolute paths, so they lead to the /build dir
            const pathStartKey = '/public/',
                relPathStart = filename.indexOf(pathStartKey) + pathStartKey.length;
            return '../../' + filename.slice(relPathStart);

        } else {
            return filename;
        }
    }

    /**
     * Cache results of a function with constant output
     * @param {Function} fn
     * @returns {Function}
     */
    function memoize(fn) {
        let cache;
        return function() {
            cache = cache || fn();
            return cache;
        };
    }

    /**
     * Defaults an argument value
     * @param argument
     * @param defaultValue
     * @returns {*}
     */
    function defaultArg(argument, defaultValue) {
        return (argument === undefined) ? defaultValue : argument;
    }

};

pr.md

This PR will allow us to write ES6 code for our browser applications, 'transpile' (transform) it into ES5 for backwards-compatibility, and use new language features in even the likes of IE9.

Live deployment

You can see this working for yourself, end to end, on (redacted URL). We're using ES6 arrow functions right on the homepage!

If you open the sources panel and navigate to all.min.js, though, you can see that this gets served to the browser as good ol' ES5:

Not only that - but the sourcemaps still point to the original source code:

And because the code is plain old ES5, it runs just fine in Internet Explorer 9:

What is 'transpilation', and how does it work?

Transpilation is a portmenteau of translate and compile. When we compile a programming language we turn its human-readable terms into assembly-level instructions for a processor to execute. By the same vein, when we transpile code we translate it into plain JavaScript for a browser to run natively.

A simple example is the new const keyword, which signifies an immutable. Once in the browser, we can just use a plain old var:

Similarly simple are arrow functions, which can (often) be implemented with a straightforward text replacement:

Some transformations are a little more complicated, though...

To make life easy, we need a third party tool. This is where Babel comes in.

Babel is a generic JavaScript transpiler. It can handle many JS-like input-languages - TypeScript and React JSX scripts among them - but ES6 transformation is its forté. By passing code through Babel and appending a special polyfill file, we can use any features in the ES6 in IE9 upwards.

Does transpiling code make files any larger?

Yes, slightly. The Babel polyfill is about 32.7kb minified and gzipped. As implemented right now, this makes our bundles generally approximately 10% larger:

File	Before size (gz, raw)	After size	Size change
app	368,645 (1,412,628)	401,423 (1,506,370)	32,728 (+8.8%)
article-preview	295,071 (1,018,121)	327,937 (1,115,129)	32,866 (+11.13%)
embedded	297,990 (1,030,705)	330,824 (1,127,583)	32,834 (+11.0%)
login	300,484 (1,047,435)	333,328 (1,144,196)	32,844 (+10.9%)
unsupported	257,759 (863,333)	290,722 (961,488)	32,963 (+12.8%)

Nevertheless - I believe this is an acceptable penalty. 30kb is the size of a medium-scale image. If filesize matters that much, there are likely better ways of saving space than eschewing ES6 - a tree-shaking JS bundler, for one.

So how is this implemented?

Our original JavaScript pipeline was simple: compile the Angular templates, concatenate all the scripts and minify the lot with uglify.

Initially, I tried to use grunt-babel after the concat stage to transform the whole bundle in one fell swoop. This worked, but was tremendously slow - building app.min.js took over two minutes! More importantly, it broke sourcemaps.

So instead, I wrote a self-contained compileJS task that wires together concatenation, transpilation and minification using 'files' that live almost entirely in-memory, caching results and making the most use I can of concurrency. By avoiding disc I/O as far as possible, compileJS can generate a bundle like app.min.js in a little over fifteen seconds:

What does this mean for developing code?

By default, the grunt start task tells the application to serve the original, unminified files. I haven't changed that. This means that you'll be running and debugging ES6 code directly in Chrome or Firefox.

One disadvantage of this, however, is that you can't directly develop in old browsers like IE9. You'll have to use the TTW_UNCOMPILED=false environment flag to run the minified, transpiled bundle. This will make debugging with browser tools a bit harder, because if the browser can't support sourcemaps you have to debug the minified code instead.

Let me know if you need do this often and we can think up some better development flow.

What about tests?

We run our unit tests on PhantomJS, a headless browser. Unfortunately this doesn't yet support ES6, so we transpile code for this instance too. This does impact coverage reports slightly - see below.

What features can I now use in my JavaScript?

You can use everything in the ES6 standard, including the new class, generator and function syntaxes.

What are the shortcomings?

Coverage reports don't reflect the original source

Our karma-coverage plugin doesn't understand ES6, so it has to be run against the transpiled ES5. This doesn't affect coverage percentages, but it does mean that when you look at the reports in detail, you'll be eyeballing ES5 rather than ES6:

This means that as we use more ES6 features in our code, these in-detail reports will become less useful. There are solutions to this - like the Istanbul plugin for Babel - but I'd like to understand exactly how much people use these reports before investing time fixing this.

Our build now relies on a big, black-box 'compileJS' program

This is unavoidable, really - Grunt's paradigm is to chain plugins that do nothing but file transformations. You can't 'communicate' between tasks except by writing temp files, and tasks run purely in serial, so implementing this over multiple tasks would be very slow indeed.

To help make the compileJS tasks a little simpler to debug, however, I've added an error handler that prints out exceptions and their stacktraces. This should help anyone who intends to do further work with this file:

You can't debug files directly in IE9/10

As mentioned above, to develop on non-ES6 browsers you'll have to use the TTW_UNCOMPILED=false environment flag, and run the minified, transpiled code locally. This will be a lot less easy to debug. Again - let me know if you think this impacts you, and maybe we can think through a solution.

How do I test this myself?

Run grunt build then TTW_UNCOMPILED=false grunt start.

The TTW_UNCOMPILED=false property tells the server to use the minified bundle rather than the individual raw files.