DataTransformUtilsV2

DataTransformUtils.js

/*
 * NOTE; This is a work in progress and has not been tested in its current form.
 * Set of methods to simplify loading data into ServiceNow.  Use with Inbound Email, rest, etc...
 * A temporary DS/importset will be created for the data load.   It will be removed if the cleanup method is called.
 * Example usage: DataTransformUtils().getDataSourceByName('Test DataSource').copyDataSource().copyAtt(sourceRecord).loadData().getMapByName('Test Transform Map').importData().cleanUpImport().log();
 *  this will:
 * 1. Retrieve Existing DS by name
 * 2. Create a clone of this datasource
 * 3. Copy attachments from a source record to copied datasource
 * 4. load data from the datasource into the import set table
 * 5 Retrieve transform map to use
 * 6. Transform loaded data
 * 7. Cleanup temporary ds
 * 8. Log the steps to the system log
 */

var DataTransformUtils = function () {
  var logger = new GSLog("cf.dataTransFormUtils.log", "DataTransformUtils");

  /**
   * create a new datasource from a mapping of field:value
   * @param {Array} dataSourceMap
   * @returns {GlideRecord} Datasource
   */
  function createDataSource(dataSourceMap) {
    _argumentValidation(
      dataSourceMap,
      _objectCheck,
      "dataSourceMap  variable must be an object containing field name and value pairs"
    );
    var grDatasource = new GlideRecord("sys_data_source");
    for (var key in dataSourceMap) {
      grDatasource.setValue(key, dataSourceMap[key]);
    }
    var ts = _getTimeStamp();
    grDatasource.name = grDS.name + ts;
    grDatasource.insert();

    logger.logDebug("Created DataSource - " + datasource.name);
    return grDatasource;
  }

  /**
   * generate timestamp from current date
   * @returns {string}
   */
  function _getTimeStamp() {
    var dateTimeObj = new GlideDateTime();
    return dateTimeObj.getByFormat("yyyyMMddHHmmss");
  }

  /**
   * Get DataSource record by sys_id
   * @param {String} sysid
   * @returns {GlideRecord}
   */
  function getDataSourceBySysID(sysid) {
    _argumentValidation(
      sysid,
      _notNil,
      "Valid sysid required"
    );
    grDatasource = _getRecord("sys_data_source", sysid);
    return grDatasource;
  }
  /**
   * get DataSource record by name
   * @param {String} name
   * @returns {GlideRecord}
   */
  function getDataSourceByName(name) {
    _argumentValidation(
      name,
      _notNil,
      "Name is required when calling getDataSourceByName"
    );

    var qry = [
      {
        field: "name",
        value: name,
      },
      {
        field: "active",
        value: true,
      },
    ];
    var grDatasource = _qryRecord("sys_data_source", qry);
    return grDatasource;
  }

  /**
   * Get Transform Map by SysID
   * @param {String} sysid
   * @returns this
   */
  function getMapBySysID(sysid) {
    _argumentValidation(
      sysid,
      _notNil,
      "Valid sysid required"
    );
    var map = _getRecord("sys_transform_map", sysid);
    _updateStatus("Retrieved Transform Map - " + map.name);
    return this;
  }

  /**
   * Get Transform Map by Name
   * @param {String} name
   * @returns {GlideRecord}
   */
  function getMapByName(name) {
    _argumentValidation(
      name,
      _notNil,
      "Transform Map name must not be empty"
    );

    var qry = [
      {
        field: "name",
        value: name,
      },
      {
        field: "active",
        value: true,
      },
    ];
    var grTransformMap = _qryRecord("sys_transform_map", qry);
    logger.logDebug("Retrieved Transform Map - " + map.name);
    return grTransformMap;
  }

  /**
   * Get GlideRecord by Sys_ID
   * @param {String} tbl
   * @param {String} sysid
   * @returns {GlideRecord}
   */
  function _getRecord(tbl, sysid) {
    _argumentValidation(
      [tbl, sysid],
      _notNil,
      "Table and sysid variables must not be empty and valid"
    );

    var grRec = new GlideRecord(tbl);
    grRec.get(sysid);
    return grRec;
  }

  /**
   * Get GlideRecord by qry
   * @param {String} tbl
   * @param {Array} queries
   * @returns {GlideRecord}
   */
  function _qryRecord(tbl, queries) {
    var qryObj;
    var exampleObj = [
      {
        field: "name",
        value: name,
      },
      {
        field: "active",
        value: true,
      },
    ];
    _argumentValidation(
      [tbl, sysid],
      _notNil,
      "Table and queries variables must not be empty\nExample Query Obj:\n" +
        JSON.stringify(exampleObj)
    );
    _argumentValidation(
      queries,
      Array.isArray,
      "Queries variable must be an Array of objects containing field name , operator, and value"
    );

    var grRec = new GlideRecord(tbl);
    for (var i = 0; i++; i < queries.length) {
      qryObj = queries[i];
      grRec.addQuery(
        qryObj["field"],
        qryObj["operator"] || "=",
        qryObj["value"]
      );
    }
    grRec.setLimit(1);
    grRec.query();

    return grRec.next() ? grRec : null;
  }

  /**
   * Copy attachments from source record to datasource
   * @param {GlideRecord} grAttSourceRecord
   * @param {GlideRecord} grDstDatasource
   * @returns {Array}
   */
  function copyAtt(grAttSourceRecord, grDatasource) {
    _argumentValidation(
      [grSrcDataSource, grDstDatasource],
      _notNil,
      "Source record and Datasource record must be valid"
    );

    var attachments = new GlideSysAttachment().copy(
      grAttSourceRecord.getRecordClassName(),
      grAttSourceRecord.getValue("sys_id"),
        grDstDatasource.getRecordClassName(),
      grDstDatasource.getValue("sys_id")
    );
    logger.logDebug("Copied Attachments from " + grAttSourceRecord.name + " to " + grDstDatasource.name);
    return attachments.join(",");
  }

  /* Legacy
    function importData(grDataSource, grTransformMap) {
        _argumentValidation([grSource,grDatasource],_notNil,"Valide DataSource GlideRecord and TrasformMap GlideRecord required");
        _loadData(grDataSource)._transformData(grTransformMap);

        return this;


    }
*/

  /**
   * Copy an existing datasource to a new DataSource for one time import
   * @param {GlideRecord} grDataSource
   * @returns this
   */
  function copyDataSource(grDataSource) {
    _argumentValidation(
      [grDatasource],
      _notNil,
      "Datasource argument required"
    );

    var dsObj = {};
    for (var key in grDataSource) {
      if (/^sys_/gi.test(key) == false) {
        dsObj[key] = grDataSource[key];
      }
    }
    logger.logDebug("Copying Datasource: " + datasource.name);

    var grNewDataSource = createDataSource(dsObj);

    return grNewDataSource;
  }

  /**
   * Transform Loaded Data
   * @param {GlideRecord} grTransformMap
   * @param {GlideRecord} grImportSet
   * @returns {GlideRecord}
   */
  function transformData(grImportSet, grTransformMap) {
    _argumentValidation(
      grImportSet,
      _notNil,
      "ImportSet argument required"
    );

    var importSetTransformer = new GlideImportSetTransformer();
    importSetTransformer.transformAllMaps(grImportSet);

    logger.logDebug("Imported Data using transform map " + transformMap.name);
    return grImportSet;
  }

  /**
   * Load Data from datasource
   * @param {GlideRecord} grDatasource DataSource
   * @param {GlideRecord} grImportSet
   * @returns
   */
  function loadData(grDatasource, grImportSet) {
    _argumentValidation(
      [grDatasource, grImportSet],
      _notNil,
      "Datasource and Importset required"
    );
    var loader = new GlideImportSetLoader();
    var importSetGr = loader.getImportSetGr(grDatasource);
    var ranload = loader.loadImportSetTable(grImportSet, grDatasource);
    if (!ranLoad) {
      throw "Failed to load import set " + grDatasource.name;
    }

    logger.logDebug("Loaded Data using dataorce " + grDatasource.name);
    return;
  }

  /**
   * Checks if variable is null, undefined, empty
   * @param varToCheck
   */
  function _argumentValidation(args, validationFunction, msg) {
    msg = msg || "All arguments required";

    for (var argIdx = 0; argIdx < arguments.length; argIdx++) {
      if (validationFunction(arguments[argIdx]) === false) {
        throw msg;
      }
    }
  }

  /**
   * Checks if variable is an Object
   * @param varToCheck
   * @returns {boolean}
   */
  function _isObject(varToCheck) {
    return typeof varToCheck === "object";
  }

  /**
   * Check if variable is null,undefined, blank
   * @param {*} varToCheck 
   * @returns {boolean}
   */
  function _notNil(varToCheck) {
    return !gs.nil(varToCheck);
  }

  function _isValidSysID(sysid) {
    return GlideStringUtil.isEligibleSysID(sysid);
  }
  /**
   * Cleans up after data load
   * @returns
   */
  function cleanImport(grImportSet) {
    if (_isEmpty(grImportSet) {
      throw "Post Import cleanup can not run until after import";
    }
    var cleaner = new ImportSetCleaner(grImportSet.getRecordClassName());
    cleaner.setDataOnly(true);
    cleaner.setDeleteMaps(false);
    cleaner.clean();
    logger.logDebug("Cleaned up import");
    return;
  }
  /*
   * Remove temporary datasource
   *returns this
   */
  function CleanUpTempData(grDatasource) {
    if (grDatasource) {
      grDatasource.deleteRecord();
    }
    return;
  }

  /*************************************
   ******* Public Methods     **********
   ************************************/
  return {
    getDataSourceBySysID: getDataSourceBySysID,
    getDataSourceByName: getDataSourceByName,
    createDataSource: createDataSource,
    copyDataSource: copyDataSource,
    copyAttToDataSource: copyAtt,
    getMapBySysID: getMapBySysID,
    getMapByName: getMapByName,
    loadData: loadData,
    importData: transformData,
    cleanUp: CleanUpTempData,
    type: "DataTransformUtils",
  };
};

pipe.js

function pipe() {
  var fns = Array.prototype.slice.call(arguments);

  return function(result) {
    return fns.reduce(function(value, fn) {
      return fn(value);
    }, result);
  };
}