serverless-webpack: multi compile option

packageModules.js

'use strict';

const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const archiver = require('archiver');
const fs = require('fs');
const glob = require('glob');
const semver = require('semver');

function setArtifactPath(funcName, func, artifactPath) {
  const version = this.serverless.getVersion();

  // Serverless changed the artifact path location in version 1.18
  if (semver.lt(version, '1.18.0')) {
    func.artifact = artifactPath;
    func.package = _.assign({}, func.package, { disable: true });
    this.serverless.cli.log(`${funcName} is packaged by the webpack plugin. Ignore messages from SLS.`);
  } else {
    func.package = {
      artifact: artifactPath,
    };
  }
}

function zip(directory, name) {
  const zip = archiver.create('zip');
  // Create artifact in temp path and move it to the package path (if any) later
  const artifactFilePath = path.join(this.serverless.config.servicePath,
    '.serverless',
    name
  );
  this.serverless.utils.writeFileDir(artifactFilePath);

  const output = fs.createWriteStream(artifactFilePath);

  const files = glob.sync('**', {
    cwd: directory,
    dot: true,
    silent: true,
    follow: true,
  });

  if (_.isEmpty(files)) {
    const error = new this.serverless
      .classes.Error('Packaging: No files found');
    return BbPromise.reject(error);
  }

  output.on('open', () => {
    zip.pipe(output);

    _.forEach(files, filePath => {
      const fullPath = path.resolve(
        directory,
        filePath
      );

      const stats = fs.statSync(fullPath);

      if (!stats.isDirectory(fullPath)) {
        zip.append(fs.readFileSync(fullPath), {
          name: filePath,
          mode: stats.mode,
          date: new Date(0), // necessary to get the same hash when zipping the same content
        });
      }
    });

    zip.finalize();
  });

  return new BbPromise((resolve, reject) => {
    output.on('close', () => resolve(artifactFilePath));
    zip.on('error', (err) => reject(err));
  });
}

module.exports = {
  packageModules() {
    const stats = this.compileStats;
    console.log(JSON.stringify(this.entryFunctions));

    return BbPromise.mapSeries(stats.stats, (compileStats, index) => {
      let entryFunction = _.find(this.entryFunctions, function(entry) {
        const compileName = entry.funcName || _.camelCase(entry.entry.key);
        return compileStats.compilation.compiler.outputPath.endsWith(compileName);
      });
      if (!entryFunction) { entryFunction = {}; }
      const filename = `${entryFunction.funcName || this.serverless.service.getServiceObject().name}.zip`;
      const modulePath = compileStats.compilation.compiler.outputPath;
      console.log(modulePath);
      console.log(filename);

      if (_.get(this.serverless, 'service.package.individually') && !entryFunction.func) {
        return BbPromise.resolve();
      }

      const startZip = _.now();
      return zip.call(this, modulePath, filename)
      .tap(() => this.options.verbose &&
        this.serverless.cli.log(`Zip ${_.isEmpty(entryFunction) ? 'service' : 'function'}: ${modulePath} [${_.now() - startZip} ms]`))
      .then(artifactPath => {
        if (_.get(this.serverless, 'service.package.individually')) {
          setArtifactPath.call(this, entryFunction.funcName, entryFunction.func, path.relative(this.serverless.config.servicePath, artifactPath));
        }
        return artifactPath;
      });
    })
    .then(artifacts => {
      if (!_.get(this.serverless, 'service.package.individually') && !_.isEmpty(artifacts)) {
        // Set the service artifact to all functions
        const allFunctionNames = this.serverless.service.getAllFunctions();
        _.forEach(allFunctionNames, funcName => {
          const func = this.serverless.service.getFunction(funcName);
          setArtifactPath.call(this, funcName, func, path.relative(this.serverless.config.servicePath, artifacts[0]));
        });
        // For Google set the service artifact path
        if (_.get(this.serverless, 'service.provider.name') === 'google') {
          _.set(this.serverless, 'service.package.artifact', path.relative(this.serverless.config.servicePath, artifacts[0]));
        }
      }

      return null;
    });
  }
};

validate.js

'use strict';

const BbPromise = require('bluebird');
const path = require('path');
const fse = require('fs-extra');
const glob = require('glob');
const lib = require('./index');
const _ = require('lodash');
const Configuration = require('./Configuration');

/**
 * For automatic entry detection we sort the found files to solve ambiguities.
 * This should cover most of the cases. For complex setups the user should
 * build his own entries with help of the other exports.
 */
const preferredExtensions = [ '.js', '.ts', '.jsx', '.tsx' ];

module.exports = {
  validate() {
    const getHandlerFile = handler => {
      // Check if handler is a well-formed path based handler.
      const handlerEntry = /(.*)\..*?$/.exec(handler);
      if (handlerEntry) {
        return handlerEntry[1];
      }
    };

    const getEntryExtension = fileName => {
      const files = glob.sync(`${fileName}.*`, {
        cwd: this.serverless.config.servicePath,
        nodir: true,
        ignore: this.configuration.excludeFiles ? this.configuration.excludeFiles : undefined
      });

      if (_.isEmpty(files)) {
        // If we cannot find any handler we should terminate with an error
        throw new this.serverless.classes.Error(
          `No matching handler found for '${fileName}' in '${this.serverless.config.servicePath}'. Check your service definition.`
        );
      }

      // Move preferred file extensions to the beginning
      const sortedFiles = _.uniq(
        _.concat(
          _.sortBy(_.filter(files, file => _.includes(preferredExtensions, path.extname(file))), a => _.size(a)),
          files
        )
      );

      if (_.size(sortedFiles) > 1) {
        this.serverless.cli.log(
          `WARNING: More than one matching handlers found for '${fileName}'. Using '${_.first(sortedFiles)}'.`
        );
      }
      return path.extname(_.first(sortedFiles));
    };

    const getEntryForFunction = (name, serverlessFunction) => {
      const handler = serverlessFunction.handler;

      const handlerFile = getHandlerFile(handler);
      if (!handlerFile) {
        _.get(this.serverless, 'service.provider.name') !== 'google' &&
          this.serverless.cli.log(
            `\nWARNING: Entry for ${name}@${handler} could not be retrieved.\nPlease check your service config if you want to use lib.entries.`
          );
        return {};
      }
      const ext = getEntryExtension(handlerFile);

      // Create a valid entry key
      return {
        [handlerFile]: `./${handlerFile}${ext}`
      };
    };

    // Initialize plugin configuration
    this.configuration = new Configuration(this.serverless.service.custom);
    this.options.verbose &&
      this.serverless.cli.log(`Using configuration:\n${JSON.stringify(this.configuration, null, 2)}`);
    if (this.configuration.hasLegacyConfig) {
      this.serverless.cli.log(
        'Legacy configuration detected. Consider to use "custom.webpack" as object (see README).'
      );
    }

    this.webpackConfig = this.configuration.config || this.configuration.webpackConfig;

    // Expose entries - must be done before requiring the webpack configuration
    const entries = {};

    const functions = this.serverless.service.getAllFunctions();
    if (this.options.function) {
      const serverlessFunction = this.serverless.service.getFunction(this.options.function);
      const entry = getEntryForFunction.call(this, this.options.function, serverlessFunction);
      _.merge(entries, entry);
    } else {
      _.forEach(functions, (func, index) => {
        const entry = getEntryForFunction.call(this, functions[index], this.serverless.service.getFunction(func));
        _.merge(entries, entry);
      });
    }

    // Expose service file and options
    lib.serverless = this.serverless;
    lib.options = this.options;
    lib.entries = entries;

    if (_.isString(this.webpackConfig)) {
      const webpackConfigFilePath = path.join(this.serverless.config.servicePath, this.webpackConfig);
      if (!this.serverless.utils.fileExistsSync(webpackConfigFilePath)) {
        return BbPromise.reject(
          new this.serverless.classes.Error(
            'The webpack plugin could not find the configuration file at: ' + webpackConfigFilePath
          )
        );
      }
      try {
        this.webpackConfig = require(webpackConfigFilePath);
      } catch (err) {
        this.serverless.cli.log(`Could not load webpack config '${webpackConfigFilePath}'`);
        return BbPromise.reject(err);
      }
    }

    // Intermediate function to handle async webpack config
    const applyDefaults = compile => {
      // Default context
      if (!compile.context) {
        compile.context = this.serverless.config.servicePath;
      }

      // Default target
      if (!compile.target) {
        compile.target = 'node';
      }

      // Default output
      if (!compile.output || _.isEmpty(compile.output)) {
        const outputPath = path.join(this.serverless.config.servicePath, '.webpack');
        compile.output = {
          libraryTarget: 'commonjs',
          path: outputPath,
          filename: '[name].js'
        };
      }

      // Custom output path
      if (this.options.out) {
        compile.output.path = path.join(this.serverless.config.servicePath, this.options.out);
      }

      if (!this.keepOutputDirectory) {
        this.options.verbose && this.serverless.cli.log(`Removing ${compile.output.path}`);
        fse.removeSync(compile.output.path);
      }
    };
    const processConfig = _config => {
      this.webpackConfig = _config;

      if (_.isArray(this.webpackConfig)) {
        // passed multiCompiler
        // @see https://webpack.js.org/api/node/#multicompiler
        let isOutputPathDifferent = false;
        _.forEach(this.webpackConfig, compile => {
          applyDefaults(compile);
          if (!this.webpackOutputPath) {
            this.webpackOutputPath = compile.output.path;
          }
          if (this.webpackOutputPath != compile.output.path) {
            isOutputPathDifferent = true;
          }
        });
        const packageIndividually = _.has(this.serverless, 'service.package') && this.serverless.service.package.individually ? true : false;
        // in case of multi compile config, user has to provide config for each function
        if (!packageIndividually && isOutputPathDifferent) {
          return BbPromise.reject(
            new this.serverless.classes.Error(
              'All multi compile config should have same output.path when package individually is false.'
            )
          );
        }
        this.multiCompile = true;
      } else {
        // single
        applyDefaults(this.webpackConfig);
        this.webpackOutputPath = this.webpackConfig.output.path;
      }

      // In case of individual packaging we have to create a separate config for each function
      if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) {
        this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)');
        this.multiCompile = true;

        if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) {
          return BbPromise.reject(
            new this.serverless.classes.Error(
              'Webpack entry must be automatically resolved when package.individually is set to true. ' +
                'In webpack.config.js, remove the entry declaration or set entry to slsw.lib.entries.'
            )
          );
        }

        // Lookup associated Serverless functions
        const allEntryFunctions = _.map(this.serverless.service.getAllFunctions(), funcName => {
          const func = this.serverless.service.getFunction(funcName);
          const handler = func.handler;
          const handlerFile = path.relative('.', getHandlerFile(handler));
          return {
            handlerFile,
            funcName,
            func
          };
        });

        this.entryFunctions = _.flatMap(entries, (value, key) => {
          const entry = path.relative('.', value);
          const entryFile = _.replace(entry, new RegExp(`${path.extname(entry)}$`), '');

          const entryFuncs = _.filter(allEntryFunctions, [ 'handlerFile', entryFile ]);
          if (_.isEmpty(entryFuncs)) {
            // We have to make sure that for each entry there is an entry function item.
            entryFuncs.push({});
          }
          _.forEach(entryFuncs, entryFunc => {
            entryFunc.entry = {
              key,
              value
            };
          });
          return entryFuncs;
        });

        // if single config provided, then only create a multi config
        if (!_.isArray(this.webpackConfig)) {
          this.webpackConfig = _.map(this.entryFunctions, entryFunc => {
            const config = _.cloneDeep(this.webpackConfig);
            config.entry = {
              [entryFunc.entry.key]: entryFunc.entry.value
            };
            const compileName = entryFunc.funcName || _.camelCase(entryFunc.entry.key);
            config.output.path = path.join(config.output.path, compileName);
            return config;
          });
        } else {
          // multi compile config count >= number of functions
          if (this.webpackConfig.length < this.entryFunctions.length) {
            return BbPromise.reject(
              new this.serverless.classes.Error(
                'Provide webpack config for each function defined.'
              )
            );
          }
          // verify that each function has entry point and output.path appended as function name
          let anyFunctionFoundWithoutConfig = false;
          _.forEach(this.entryFunctions, entryFunc => {
            const functionConfig = this.webpackConfig.find(config => {
              const compileName = entryFunc.funcName || _.camelCase(entryFunc.entry.key);
              if (_.isObjectLike(config.entry)) {
                return _.isEqual(config.entry, {[entryFunc.entry.key]: entryFunc.entry.value})
                  && config.output.path.endsWith(compileName);
              } else if (_.isString(config.entry)) {
                return config.entry === entryFunc.entry.value 
                  && config.output.path.endsWith(compileName);
              }
            });
            if (!functionConfig) {
              anyFunctionFoundWithoutConfig = true;
            }
          });
          if (anyFunctionFoundWithoutConfig) {
            return BbPromise.reject(
              new this.serverless.classes.Error(
                'Provide webpack config for each function with correct entry and webpack compile output path. ' +
                'Automatic webpack entry detection is disabled with user provided multi config and package ' +
                'individually is set.'
              )
            );
          }
          // finally set the this.webpackOutputPath
          this.webpackOutputPath
            = this.webpackConfig[0].output.path.substring(0, this.webpackConfig[0].output.path.lastIndexOf('/'));
        }
      } else {
        if (_.isArray(this.webpackConfig)) {
          _.forEach(this.webpackConfig, config => {
            config.output.path = path.join(config.output.path, 'service');
          });
        } else {
          this.webpackConfig.output.path = path.join(this.webpackConfig.output.path, 'service');
        }
      }

      if (this.skipCompile) {
        this.serverless.cli.log('Skipping build and using existing compiled output');
        if (!fse.pathExistsSync(this.webpackOutputPath)) {
          return BbPromise.reject(new this.serverless.classes.Error('No compiled output found'));
        }
        this.keepOutputDirectory = true;
      }

      return BbPromise.resolve();
    };

    // Webpack config can be a Promise, If it's a Promise wait for resolved config object.
    if (this.webpackConfig && _.isFunction(this.webpackConfig.then)) {
      return BbPromise.resolve(this.webpackConfig.then(config => processConfig(config)));
    } else {
      return processConfig(this.webpackConfig);
    }
  }
};