manualBundling.js

/*
See the readme for detailed guidance, as well as inline comments here.
*/

const Builder = require("systemjs-builder"),
  gulp = require("gulp"),
  gulpIf = require("gulp-if"),
  gulpUglify = require("gulp-uglify"),
  gulpRename = require("gulp-rename"),
  gulpPlumber = require("gulp-plumber"),
  sourcemaps = require("gulp-sourcemaps"),
  fs = require("fs"),
  os = require("os"),
  extend = require("extend"),
  colors = require("colors/safe"),
  ui = require("./util/ui.js"),
  fileHelpers = require("./util/fileHelpers.js"),
  Promise = require("promise"),
  systemJsPathsObj = require("../src/framework/SystemJsSetup.js"),
  md5 = require("md5"),
  coverageChecker = require("./coverageCheckerLocal");

// Map well-known libs to their pre-minified versions.
["mobx"].forEach(key => {
  systemJsPathsObj.map[key] = systemJsPathsObj.map[key] + ".min";
});

["reactRaw", "react-dom"].forEach(key => {
  systemJsPathsObj.map[key] = systemJsPathsObj.map[key].replace(/\.development/, ".production.min");
});

global.Promise = Promise; //systemJS depends on Promise existing, which it won't for Node 0.X that everyone but me is probably running

// Intercept calls to SystemJS resolver normalize to handle our special legacy cases. We do this in loaderSetup.js as well at runtime.
var systemNormalize = System.normalize; // see https://github.com/systemjs/systemjs/blob/master/src/resolve.js#L65
System.normalize = function(name, parentName, parentAddress) {
  if (/^text!/.test(name)) return Promise.resolve(name.substr(5) + "!text");
  else if (/^css!/.test(name)) return Promise.resolve(name.substr(4) + "!css");
  else if (/^load!/.test(name)) {
    return Promise.resolve(name.substr(5).replace(/-/g, "/") + ".js");
  } else return systemNormalize.call(this, name, parentName, parentAddress);
};

systemJsPathsObj.defaultJSExtensions = true;
systemJsPathsObj.sourceMaps = true;
systemJsPathsObj.sourceMapContents = true;

const crBuildFile = require("./config/build-cr.js"),
  moduleBuildFile = require("./config/build-module.js"),
  utilBuildFile = require("./config/build-utility.js"),
  buildTargets = require("./buildTargets.js");

// this is used to ignore modules we know are included in more than one bundle, and that we're okay with doing that for
const duplicatedModulesWhitelist = new Set(buildTargets.dupWhitelist.map(s => s.toLowerCase()));

/**
 * executeBuild is this module's default export is invoked by build shell.
 *
 * baseDirectory - target output base directory
 * options - mirrors what is passed from command line
 *
 * */
const executeBuild = async function executeBuild(baseDirectory, options = {}) {
  let bundleOnly = options.bundleOnly;
  let debugMode = options.debug;
  let debugStrings = [];
  let actualBundles = []; // will be populated by each call to 'bundleThem' with list of actual bundles

  var defaultRequireParams = { baseUrl: "../" + baseDirectory }; // used to tell SystemJS where to find modules

  var mainBuildInfo = extend(true, {}, buildTargets),
    emptyConfig = extend(true, {}, defaultRequireParams, systemJsPathsObj),
    crConfig = extend(true, {}, defaultRequireParams, systemJsPathsObj, crBuildFile),
    moduleConfig = extend(true, {}, defaultRequireParams, systemJsPathsObj, moduleBuildFile),
    utilConfig = extend(true, {}, defaultRequireParams, systemJsPathsObj, utilBuildFile);

  shapeConfigObject(crConfig);
  shapeConfigObject(moduleConfig, { additionalExcludes: buildTargets.labelsUtilities });
  shapeConfigObject(utilConfig, { additionalExcludes: buildTargets.labelsUtilities });

  ui.displayMessage(`Auto-detecting submodules`, { color: colors.cyan });
  try {
    await expandCompleteModules(mainBuildInfo, baseDirectory, moduleConfig);
  } catch (err) {
    console.error("Unhandled problem while generating bundles to build.", err);
    return;
  }

  ui.displayMessage(`Submodules detected and queued. Beginning bundling`, { color: colors.cyan });

  try {
    await bundleThem([], emptyConfig, { baseDirectory, actualBundles, debugMode, debugStrings }, mainBuildInfo.assetDependencies);
  } catch (err) {
    console.error("Unhandled problem bundling core assets bundles.", err);
    return;
  }

  await bundleThem(
    ["framework/cr + framework/globalUtils/scheduling - framework/koMapper/mapper - app/**/*.js"],
    crConfig,
    { baseDirectory, actualBundles, debugMode, debugStrings },
    mainBuildInfo.crDependencyBundles
  );

  await bundleThem(
    mainBuildInfo.modulesToBuild,
    moduleConfig,
    { baseDirectory, actualBundles, debugMode, debugStrings },
    mainBuildInfo.moduleDependencyBundles
  );

  await bundleThem(
    mainBuildInfo.utilsToBuild,
    utilConfig,
    { baseDirectory, actualBundles, debugMode, debugStrings },
    mainBuildInfo.utilityDependencyBundles
  );

  let unbundledFiles = await coverageChecker(actualBundles);

  if (unbundledFiles.length) {
    console.log("");
    console.log(colors.yellow("The following files were found to be un-bundled, and are being added to individual bundles"));
    unbundledFiles.forEach(file => {
      console.log(colors.yellow(file));
    });
    console.log("\n\n");

    await bundleThem(
      [],
      crConfig,
      {
        baseDirectory,
        actualBundles,
        debugMode,
        debugStrings
      },
      unbundledFiles.map(file => ({
        saveTo: file
          .replace(/\\/g, "/")
          .replace(/\//g, "-")
          .replace(/\./g, "-"),
        what: /\.css$/.test(file) ? file + "!css" : /\.htm$/.test(file) ? file + "!text" : `[${file}]`
      }))
    );
  }

  if (debugMode) {
    fs.writeFileSync(`build-debug-info.txt`, debugStrings.join(os.EOL));
  }

  let gulpExclusions = actualBundles.filter(bundle => bundle.unchanged).map(bundle => `!../${bundle.preprocessedPath}`);
  if (gulpExclusions.length !== actualBundles.length) {
    const prodBuild = !debugMode && !bundleOnly;
    ui.displayMessage("Unminified files finished building." + prodBuild ? " Minifying them now..." : "", { color: colors.green });

    let errors = false;
    let allErrors = [];
    return new Promise((resolve, reject) => {
      gulp
        .src([`../${baseDirectory}/**/**-build-unminified.js`].concat(gulpExclusions), { base: "./" })
        .pipe(
          gulpPlumber({
            errorHandler: function(error) {
              console.log(colors.red(`\nError processing ${error.fileName}: ${error.message}\n\n`));
              errors = true;
              allErrors.push(error);
              this.emit("end");
            }
          })
        )
        .pipe(sourcemaps.init({ loadMaps: true }))
        .pipe(gulpIf(prodBuild, gulpUglify())) // minify if prod build; we still want to rename
        .pipe(
          gulpRename(function(path) {
            path.basename = path.basename.replace(/build-unminified$/, "build");
            console.log(colors.green(`${path.basename} processed.`));
          })
        )
        .pipe(sourcemaps.write(""))
        .pipe(gulp.dest(""))
        .on("end", () => {
          if (errors) {
            reject(allErrors.join(", "));
          } else {
            ui.displayMessage("Processing bundles complete.", { color: colors.green });
            Promise.resolve(generateConfigFilesForClient(baseDirectory, actualBundles))
              .then(resolve)
              .catch(err => {
                console.error("Unhandled problem while generating client config files.");
                reject(err);
              });
          }
        });
    }).catch(err => {
      console.error("Unhandled problem while processing.", err);
    });
  }
  //endregion
};

/**
 * Bundles the given JS modules and their dependencies. Will push bundle info for each bundle onto the 'actualBundles' parameter.
 *
 * @param {string} modulesToBundle - bundle expression of JS modules to bundle
 * @param {*} bundlerConfig - bundle set configuration
 * @param {*} options - extra options/config (might break these out at some point; seems kinda random)
 * @param {*} preliminaryDependencies - bundle expression of dependent JS modules
 */
const bundleThem = async function bundleThem(
  modulesToBundle, // JS modules, not CR modules
  bundlerConfig,
  { baseDirectory, actualBundles, debugMode, debugStrings },
  preliminaryDependencies = []
) {
  for (let dependencyBundle of preliminaryDependencies) {
    let builder = new Builder("../" + baseDirectory);
    builder.config(bundlerConfig);

    let name = `moduleDependency: ${dependencyBundle.what}`; // for console printing/human reference
    dependencyBundle.what = dependencyBundle.what.replace(/@(\S*)/g, (s, jsModule) => `dist/moduleDependencyBundles/${jsModule}-build-unminified.js`);

    let bundle = {
      name,
      outputPath: `dist/moduleDependencyBundles/${dependencyBundle.saveTo}-build.js`,
      preprocessedPath: `dist/moduleDependencyBundles/${dependencyBundle.saveTo}-build-unminified.js`,
      friendlyName: dependencyBundle.saveTo
    };
    let bundlerOutputPath = `../${baseDirectory}/dist/moduleDependencyBundles/${dependencyBundle.saveTo}-build-unminified.js`;

    // bundle already! :)
    await builder
      .bundle(dependencyBundle.what, bundlerOutputPath, bundlerConfig)
      .then(result => {
        // try reading the file to ensure it's there/has no probs
        let newContents = fs.readFileSync(bundlerOutputPath, { encoding: "utf8" });
        console.log(colors.green(`${name} bundled`));
        // log which modules were bundled into this bundle
        bundle.bundledModules = result.modules;
        // add it to our master list of bundles
        actualBundles.push(bundle);
      })
      .catch(err => {
        console.log(colors.red("\nError in:", dependencyBundle.what, "---->", err, "\n\n"));
        throw err;
      });
  }

  for (let bundleExpression of modulesToBundle) {
    let builder = new Builder("../" + baseDirectory);
    builder.config(bundlerConfig);

    bundleExpression = bundleExpression.replace(/@(\S*)/g, (s, realName) => `dist/moduleDependencyBundles/${realName}-build-unminified.js`);

    if (debugMode) {
      debugStrings.push(bundleExpression);
    }

    let outputPathName = "";
    try {
      outputPathName = bundleExpression.split(" ")[0].trim();
      if (outputPathName.length === 0) {
        throw new Error("Output path name is blank. This may happen if you use jsGlob in your bundle definitions in buildTargets.completeModules.");
      }
    } catch (err) {
      console.error(err);
      continue;
    }

    let bundle = {
        name: outputPathName,
        outputPath: `dist/${bundlerConfig.map[outputPathName] || outputPathName}-build.js`,
        preprocessedPath: `dist/${bundlerConfig.map[outputPathName] || outputPathName}-build-unminified.js`
      },
      bundlerOutputPath = `../${baseDirectory}/dist/${bundlerConfig.map[outputPathName] || outputPathName}-build-unminified.js`;

    try {
      const bundlerResult = await builder.bundle(`${bundleExpression}`, bundlerOutputPath, bundlerConfig);

      let newContents = fs.readFileSync(bundlerOutputPath, { encoding: "utf8" });
      console.log(colors.green(`${bundleExpression} bundle bundled`));
      bundle.bundledModules = bundlerResult.modules;
      actualBundles.push(bundle);

      if (debugMode) {
        debugStrings.push(
          ...bundlerResult.modules
            .concat()
            .sort()
            .map(s => `\t${s}`),
          Array.from({ length: 100 }, s => "_").join("")
        );
      }
    } catch (err) {
      console.log(colors.red("\n", bundleExpression, "--->", err, "\n"));
      throw err;
    }
  }
};

function generateConfigFilesForClient(baseDirectory, bundles) {
  bundles.forEach(bundle => {
    bundle.outputPath += "?version=" + md5(fs.readFileSync("../dist/" + bundle.outputPath, { encoding: "utf8" }));
  });

  let bundleContents = bundles
    .sort(nameComparer)
    .map(bundle => `'${bundle.outputPath}': [${bundle.bundledModules.map(d => `'${d}'`).join(", ")}]`)
    .join(",\n\t");

  fs.writeFileSync(`../dist/${baseDirectory}/framework/cr.compressed-paths.js`, fileHelpers.compressedPathsFileContent(bundleContents));

  let builder = new Builder("../" + baseDirectory);
  builder.config({
    meta: {
      "framework/loaderSetup.js": {
        deps: ["dist/framework/cr.compressed-paths.js"]
      },
      "dist/framework/cr.compressed-paths.js": { format: "global" }
    }
  });

  return builder
    .buildStatic("framework/loaderSetup.js", `../${baseDirectory}/dist/framework/loaderSetup-build.js`)
    .then(() => {
      ui.displayMessage("Loader config set", { color: colors.green });
      checkForDuplicatedModules(bundles);
    })
    .catch(err => {
      console.error("Unhandled problem generating client config.", err);
      throw err;
    });
}

/**
 * Gets the file size of the given file.
 *
 * @param {string} file - file path to check
 */
function fileSize(file) {
  file = file.replace(/!.*$/, "").replace(/\?version=.*/, "");

  if (!/.htm$/.test(file) && !/.css$/.test(file)) {
    file = file.replace(/\.js$/, "") + ".js";
  }
  let stats = fs.statSync("../dist/" + file),
    fileSizeInBytes = stats.size;

  const MB = 1048576,
    KB = 1024;
  if (fileSizeInBytes > MB) {
    return (fileSizeInBytes / MB).toFixed(2) + "MB";
  } else if (fileSizeInBytes > KB) {
    return (fileSizeInBytes / KB).toFixed(2) + "KB";
  } else {
    return fileSizeInBytes + "B";
  }
}

/**
 * Will examine all modules in all bundles and will report any non-whitelisted duplicates.
 *
 * @param {*} bundles - full list of bundles created by the build process
 */
function checkForDuplicatedModules(bundles) {
  let allBundledModules = {}; // create a simple object map to track how many times a module is included
  bundles.forEach(bundle => {
    bundle.bundledModules.forEach(bundledModule => {
      bundledModule = bundledModule.toLowerCase();
      if (!allBundledModules[bundledModule]) {
        allBundledModules[bundledModule] = [];
      }
      allBundledModules[bundledModule].push({ name: bundle.friendlyName || bundle.name, path: bundle.outputPath });
    });
  });

  let duplicatedModules = Object.keys(allBundledModules)
    .filter(moduleName => allBundledModules[moduleName].length > 1) // filter out any that only exist once
    .filter(moduleName => !duplicatedModulesWhitelist.has(moduleName)); // ignore whitelisted duplicates

  if (duplicatedModules.length) {
    let message = "\n\nWARNING: The following dependencies are duplicated:\n\n";
    const duplicatedLocations = moduleName =>
      allBundledModules[moduleName]
        .map(bundledModule => `${bundledModule.name} at ${bundledModule.path.replace(/\?version=.*/, "")} - ${fileSize(bundledModule.path)}`)
        .join("\n\t\t");

    duplicatedModules.forEach(moduleName => (message += `${moduleName} - ${fileSize(moduleName)}\n\t\t${duplicatedLocations(moduleName)}\n\n`));
    ui.displayMessage(message, { separator: "", color: colors.yellow });
  }
}

/**
 * Based on the buildTargets configuration, this will cycle through all modules and submodules and specify the actual bundles we want to build for the modules while trying to stay within the maximum size for each.
 * @param {*} buildTargets - should be the parsed JSON from the buildTargets.js file
 * @param {*} baseDirectory - base directory to find modules and submodules in for the build
 * @param {*} configToUse - passed to SystemJS Builder--should be the same as target; in this method, we just use Builder to get sizes of bundles.
 */
const expandCompleteModules = async function expandCompleteModules(buildTargets, baseDirectory, configToUse) {
  const maxBundleSize = 204800; // discussed with Adam; he said most recent best practice is to keep these under 200k for maximum loading efficiency
  const includeModulesExpression = (...args) => args.filter(arg => arg).join(" + "); // this is just used to create bundle expressions for the Builder as we determine how we want to bundle; filters out falsey
  // NOTE: System JS "bundle arithmetic expressions" provide a DSL to describe how you want things bundled: https://github.com/systemjs/builder#bundle-arithmetic
  //       So when you see expressions like "thing + thing - thing" here, that's that's going on

  const expandPlaceholders = bundleExpression => {
    const placeHolderToExpression = placeHolderName => buildTargets.moduleDependencyBundles.find(dep => dep.saveTo === placeHolderName).what;
    let expandedExpression = bundleExpression;
    while (expandedExpression.indexOf("@") >= 0) {
      expandedExpression = expandedExpression.replace(/@(\S*)/g, (s, realName) => `( ${placeHolderToExpression(realName)} )`);
    }
    return expandedExpression;
  };

  /**
   * If dependencyExpression is truthy, it will add it as a modeulDependencyBundle.
   *
   * @param {string} rootModuleName - name of current root module
   * @param {*} bundleQualifier - some bundle name qualifier to uniquify the bundle in the module
   * @param {*} dependencyExpression - actual expression to bundle
   */
  const addModuleDependencyBundle = (rootModuleName, bundleQualifier, dependencyExpression) => {
    if (dependencyExpression) {
      buildTargets.moduleDependencyBundles.push({
        what: dependencyExpression,
        saveTo: `${rootModuleName}-${bundleQualifier}-bundle`
      });
    }
  };

  let modules = Object.keys(buildTargets.completeModules);
  for (let moduleCounter = 0; moduleCounter < modules.length; moduleCounter++) {
    const rootModule = modules[moduleCounter],
      modulePath = `../${baseDirectory}/${rootModule}`;
    const moduleBuildInfo = buildTargets.completeModules[rootModule],
      subModuleExclusions = moduleBuildInfo.excludeSubModules || [],
      __bundleWithRoot = moduleBuildInfo.__bundleWithRoot;
    delete moduleBuildInfo.__bundleWithRoot;

    let bundleWithRootModule = __bundleWithRoot ? " + " + __bundleWithRoot : "",
      excludeBundledWithRoot = __bundleWithRoot ? " - ( " + __bundleWithRoot + " ) " : "";

    let jsBundleExpression = expandPlaceholders((moduleBuildInfo.root || `${rootModule}/${rootModule}`) + bundleWithRootModule),
      htmlBundleExpression = fs.existsSync(`${modulePath}/${rootModule}.htm`) ? `${rootModule}/${rootModule}.htm!text` : null,
      cssBundleExpression = fs.existsSync(`${modulePath}/${rootModule}.css`) ? `${rootModule}/${rootModule}.css!css` : null;

    let builder = new Builder("../" + baseDirectory);
    builder.config(configToUse);

    try {
      const jsMinified = await builder.bundle(jsBundleExpression, { minify: true });

      let jsSize = jsMinified.source.length,
        htmlSize = 0,
        cssSize = 0;

      if (htmlBundleExpression) {
        const html = await builder.bundle(htmlBundleExpression, {});
        htmlSize = html.source.length;
      }

      if (cssBundleExpression) {
        const css = await builder.bundle(cssBundleExpression, {});
        cssSize = css.source.length;
      }

      if (jsSize + htmlSize + cssSize < maxBundleSize) {
        buildTargets.modulesToBuild.push(includeModulesExpression(jsBundleExpression, htmlBundleExpression, cssBundleExpression));
      } else if (jsSize + htmlSize < maxBundleSize) {
        addModuleDependencyBundle(rootModule, "css", cssBundleExpression);
        buildTargets.modulesToBuild.push(includeModulesExpression(jsBundleExpression, htmlBundleExpression));
      } else if (jsSize + cssSize < maxBundleSize) {
        addModuleDependencyBundle(rootModule, "html", htmlBundleExpression);
        buildTargets.modulesToBuild.push(includeModulesExpression(jsBundleExpression, cssBundleExpression));
      } else {
        buildTargets.modulesToBuild.push(jsBundleExpression);

        if (htmlSize && cssSize && htmlSize + cssSize < maxBundleSize) {
          buildTargets.moduleDependencyBundles.push({
            what: includeModulesExpression(htmlBundleExpression, cssBundleExpression),
            saveTo: `${rootModule}-html-css-bundle`
          });
        } else {
          addModuleDependencyBundle(rootModule, "css", cssBundleExpression);
          addModuleDependencyBundle(rootModule, "html", htmlBundleExpression);
        }
      }

      let subFolders = fs.readdirSync(modulePath).filter(file => file != "langs" && fs.statSync(`${modulePath}/${file}`).isDirectory());

      for (let subModuleCounter = 0; subModuleCounter < subFolders.length; subModuleCounter++) {
        let submoduleName = subFolders[subModuleCounter];
        try {
          submoduleName = submoduleName.toLowerCase();
          if (subModuleExclusions.indexOf(submoduleName) >= 0) continue; // can configure to exclude submodule folders in buildTargets.js using excludeSubModules

          let jsBundleExpression;

          if (moduleBuildInfo.hasOwnProperty(submoduleName)) {
            jsBundleExpression = moduleBuildInfo[submoduleName] + excludeBundledWithRoot;
          } else {
            if (!fs.existsSync(`${modulePath}/${submoduleName}/${submoduleName}.js`)) {
              continue;
            }
            jsBundleExpression = `${rootModule}/${submoduleName}/${submoduleName}` + excludeBundledWithRoot;
          }

          jsBundleExpression = expandPlaceholders(jsBundleExpression);

          if (!fs.existsSync(`${modulePath}/${submoduleName}/${submoduleName}.htm`)) {
            buildTargets.modulesToBuild.push(jsBundleExpression);
          } else {
            const bundledJS = await builder.bundle(jsBundleExpression, { minify: true }),
              jsBundleSize = bundledJS.source.length,
              htmlBundleExpression = `${modulePath}/${submoduleName}/${submoduleName}.htm!text`,
              bundledHtml = await builder.bundle(htmlBundleExpression, {}),
              htmlBundleSize = bundledHtml.source.length;
            if (jsBundleSize + htmlBundleSize < maxBundleSize) {
              buildTargets.modulesToBuild.push(includeModulesExpression(jsBundleExpression, htmlBundleExpression));
            } else {
              addModuleDependencyBundle(rootModule, `${submoduleName}-html`, htmlBundleExpression);
              buildTargets.modulesToBuild.push(jsBundleExpression);
            }
          }
        } catch (err) {
          console.error(`Error expanding subModule '${submoduleName}'.`, err);
        }
      }
    } catch (err) {
      console.error(`Error expanding module '${rootModule}'.`, err);
    }
  }
};

function shapeConfigObject(config, overrides = {}) {
  if (config.excludeDirs) {
    config.excludeDirs.forEach(dir => config.exclude.push(`${dir}/*`));
  }
  config.meta = config.meta || {};
  (config.exclude || []).concat(overrides.additionalExcludes || []).forEach(ex => (config.meta[ex] = { build: false }));

  config.sourceMaps = false;
  config.sourceMapContents = false;
}

function nameComparer(obj1, obj2) {
  if (obj1.name < obj2.name) return -1;
  if (obj1.name > obj2.name) return 1;
  return 0;
}

module.exports = executeBuild;