bm_klevu/cartridge/scripts/jobs/export/klevuContentAssetsExport.js

contentAssetsUtils.js

/* bm_klevu/cartridge/scripts/utils/contentAssetsUtils.js */
'use strict';

/* API Includes*/
const File = require('dw/io/File');
const FileWriter = require('dw/io/FileWriter');
const FileReader = require('dw/io/FileReader');
const XMLIndentingStreamWriter = require('dw/io/XMLIndentingStreamWriter');
const XMLStreamReader = require('dw/io/XMLStreamReader');
const XMLStreamConstants = require('dw/io/XMLStreamConstants');
const Calendar = require('dw/util/Calendar');
const StringUtils = require('dw/util/StringUtils');
const URLUtils = require('dw/web/URLUtils');
const Site = require('dw/system/Site');

/* Script Includes*/
const klevuUtils = require('~/cartridge/scripts/utils/klevuUtilsBm');
const klevuJobUtils = require('~/cartridge/scripts/utils/klevuJobUtils');
const LogUtils = require('*/cartridge/scripts/utils/klevuLogUtils');
const Logger = LogUtils.getLogger('contentAssetsUtils');

/* Global Variables*/
var config = klevuUtils.config;
var timeStamp = StringUtils.formatCalendar(new Calendar(new Date()), 'yyyyMMddHHmmss');
var fw;
var file;
var fileType;
var fullFileName;
var counter = 0;
var filesCounter = 0;
var fileNamesList = [];
var cmsRecordsWrittenSuccessfully = 0;

/**
 * Function to retrieve content attributes
 * @return {Array} all the required fields
 */
function getContentAttributes() {
    var requiredFields = [];
    requiredFields.push('id', 'item_type', 'title', 'link', 'published_at', 'description');

    return requiredFields;
}

/**
 * Function to calculate content attributes values
 * @param {dw.content.Content} content - contentasset
 * @return {Object} all the required fields
 */
function calculateContentFields(content) {
    var itemType = config.klevuCMS;
    var contentID = content ? (config.contentPrefix + content.ID) : '';
    var link = URLUtils.https(config.pageShow, config.cid, content.ID);
    var title = content ? content.name : '';
    var publishedDate = content ? StringUtils.formatCalendar(new Calendar(content.creationDate), 'YYYY-MM-dd\'T\'HH:mm:ss.SSSZ') : '';
    var contentBody = (content && content.custom.body) ? content.custom.body.source : '';

    Logger.info(contentID);

    return {
        id: contentID,
        item_type: itemType,
        title: title,
        link: link,
        published_at: publishedDate,
        description: contentBody
    };
}

/**
 * Function to check and delete empty file
 * @param {dw.io.File} file - object
 * @returns {void}
 */
function checkEmptyFile(file) {
    var fr = new FileReader(file, 'UTF-8');
    var xsr = new XMLStreamReader(fr);
    var isItemElementPresent = false;

    while (xsr.hasNext()) {
        if (xsr.next() === XMLStreamConstants.START_ELEMENT) {
            var localElementName = xsr.getLocalName();

            if (localElementName === 'item') {
                isItemElementPresent = true;
                break;
            }
        }
    }

    if (!isItemElementPresent) {
        file.remove();
    }
}

/**
 * Utility class for content assets
 */
let contentAssetsUtils = {
    /**
     * Function for delta content export
     * @param {List} content - object
     * @returns {Object} calculated fields values
     */
    deltaExport: function (content) {
        let customCache = require('~/cartridge/scripts/utils/customCacheWebdav');
        let SEP = File.SEPARATOR;
        let objectTypeName = config.contentObject;
        let endPoint = Site.getCurrent()
            .getID() ? Site.getCurrent()
            .getID() + SEP + objectTypeName : objectTypeName;
        var response = null;

        if (content.lastModified.getTime() > customCache.getCacheTime(endPoint) && content.onlineFlag) {
            response = calculateContentFields(content);
        }

        return response;
    },

    /**
     * Function to get fields
     * @param {List} content - object
     * @returns {Object} calculated fields values
     */
    getContentFields: function (content) {
        var response = calculateContentFields(content);

        return response;
    },

    /**
     * Function that writes data to xml file
     * @param {Collection} lines - Actual payload for third party system
     * @param {dw.io.XMLStreamWriter} xsw - xml file
     * @param {Object} csvWriter - csv file
     * @param {string | boolean} isFullExport - determines id full export or delta export
     * @returns {Object}
     */
    writeData: function (lines, xsw, csvWriter, isFullExport) {
        var allFields = getContentAttributes();
        var fileClosed = false;
        var recordsInSingleFile = Site.current.getCustomPreferenceValue('klevuProductsInSingleFile');
        recordsInSingleFile = recordsInSingleFile < 100 ? 100 : recordsInSingleFile;

        for (let i = 0; i < lines.length; i++) {
            if (counter === 0) {
                xsw = this.createXml(filesCounter++, isFullExport, config.contentObject);
                fileClosed = false;
            }

            let line = lines[i];
            let j;

            if (line && xsw !== null) {
                xsw.writeStartElement('item');

                for (j = 0; j < allFields.length; j++) {
                    xsw.writeStartElement(allFields[j]);
                    xsw.writeCharacters(line[allFields[j]]);
                    xsw.writeEndElement();
                }

                xsw.writeEndElement();
                csvWriter.writeLine(line.id);
            }
            counter++;
            if (counter === recordsInSingleFile) {
                this.closeXml(xsw, isFullExport);
                counter = 0;
                fileClosed = true;
            }
            cmsRecordsWrittenSuccessfully++;
        }
        return {
            fileClosed: fileClosed,
            xsw: xsw,
            cmsCount: cmsRecordsWrittenSuccessfully
        };
    },

    /**
     * Function that creates the xml file
     * @param {number} count - no. of files count
     * @param {boolean} isFullExport - full export or delta export
     * @param {string} objectType - objectType
     * @returns {dw.io.XMLStreamWriter} xsw - xml file
     */
    createXml: function (count, isFullExport, objectType) {
        fileType = isFullExport ? config.contentFullExport : config.contentFullExport;
        var fileCount = count >= 0 ? (fileType + count + '-') : fileType;
        var folderPath = klevuUtils.getKlevuPath(config.fileUploadPath);
        var fileName = String.concat(config.filePrepend, fileCount, timeStamp, '.xml');
        var exportMode = isFullExport ? config.jobTypeFull : config.jobTypeDelta;
        fullFileName = folderPath + File.SEPARATOR + fileName;

        if (fileNamesList.indexOf(fileName) === -1) {
            klevuJobUtils.pushDataFileName(fileName, exportMode, objectType);
            fileNamesList.push(fileName);
        }
        file = new File(fullFileName);
        var xsw;
        var folder = new File(folderPath);

        if (!folder.exists()) {
            try {
                folder.mkdirs();
            } catch (e) {
                Logger.error(e.getMessage());
                Logger.error('Error while making folders for the file: ' + fileName);
            }
        }

        if (!file.exists()) {
            file.createNewFile();
        }

        try {
            fw = new FileWriter(file, 'UTF-8');
            xsw = new XMLIndentingStreamWriter(fw);
            // XML definition & first node
            xsw.writeStartDocument();
            xsw.writeStartElement('rss');
            xsw.writeAttribute('version', '2.0');
            xsw.writeAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance');
            xsw.writeAttribute('xsi:noNamespaceSchemaLocation', 'schema.xsd');
            xsw.writeStartElement('channel');
        } catch (e) {
            throw new Error('ERROR : While writing Xml content asset file : ' + e.stack + ' with Error: ' + e.message);
        }

        return xsw;
    },


    /**
     * Function that closes the xml file
     * @param {dw.io.XMLStreamWriter} xsw - xml file
     * @param {boolean} isFullExport - full export or delta export
     * @returns {void}
     */
    closeXml: function (xsw, isFullExport) {
        try {
            //Check if XSW is already available
            if (xsw !== null) {
                // close xml Library
                xsw.writeEndElement();
                xsw.writeEndElement();
                xsw.writeEndDocument();
                xsw.flush();
                xsw.close();
            }

            if (fw !== null) {
                fw.close();
            }

            if (!isFullExport) {
                checkEmptyFile(file);
            }

            if (file.file) {
                klevuUtils.compressFile(fullFileName);
            }
        } catch (exception) {
            Logger.error('ERROR : closing and compressing the Content Asset xml file ' + exception.stack + ' with Error: ' + exception.message);
        }
    }

};

module.exports = contentAssetsUtils;

klevuContentAssetsDeltaExport.js

/* bm_klevu/cartridge/scripts/jobs/export/klevuContentAssetsDeltaExport.js */

/* global empty */
/* eslint consistent-return:0 */

'use strict';

/* API Includes */
const Site = require('dw/system/Site');
const StringUtils = require('dw/util/StringUtils');
const Calendar = require('dw/util/Calendar');

/* Script Includes*/
const CsvWriter = require('~/cartridge/scripts/utils/csvWriter');
const klevuUtils = require('~/cartridge/scripts/utils/klevuUtilsBm');
const LogUtils = require('*/cartridge/scripts/utils/klevuLogUtils');
const Logger = LogUtils.getLogger('klevuContentAssetsDeltaExport');
const contentUtils = require('~/cartridge/scripts/utils/contentAssetsUtils');

/* Global Variables*/
var xsw;
var csvWriter;
var exportModel;
var isFullExport = false;
var config = klevuUtils.config;
var writeDataResponse;
var fileClosed = false;
var cmsCount;

/**
 * beforeStep. Creation of the file and initialization XML file
 * @returns {void}
 */
function beforeStep() {
    csvWriter = new CsvWriter(config.sentContentHeader);
    csvWriter.initializeCSVStreamWriter(
        StringUtils.format(config.baseKlevuPath + config.sentContentPath, Site.getCurrent().ID),
        StringUtils.format(config.sentContentFileName, StringUtils.formatCalendar(new Calendar(), 'yyyyMMddHHmmssSSS'))
    );
    var ExportModel = require('~/cartridge/scripts/models/contentAssets/klevuContentAssetExportModel.js');
    exportModel = new ExportModel();
}

/**
 * read. Read all the content assets
 * @returns {Array} Array of content objects
 */
function read() {
    try {
        return exportModel.getNextItem();
    } catch (exception) {
        Logger.error('ERROR :retrieving CMS data while reading content assets ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * process. Retrieve all the needed data for the XML file
 * @param {dw.content.Content} record - object
 * @returns {Array} Array of required fields
 */
function process(record) {
    try {
        return contentUtils.deltaExport(record);
    } catch (exception) {
        Logger.error('ERROR :writing CMS data ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * write. Write the data in the file
 * @param {Collection} lines - Actual payload for 3rd party system
 * @returns {void}
 */
function write(lines) {
    try {
        if (!empty(lines) && lines.length) {
            Logger.info('content assets count sent to write in xml : ' + lines.length);
            writeDataResponse = contentUtils.writeData(lines, xsw, csvWriter, isFullExport);
            xsw = writeDataResponse.xsw;
            fileClosed = writeDataResponse.fileClosed;
            cmsCount = writeDataResponse.cmsCount;
        }
    } catch (exception) {
        Logger.error('ERROR :writing CMS data ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * afterStep.
 * @param {boolean} success - XML creation successful or not
 * @returns {void}
 */
function afterStep(success) {
    if (fileClosed === false && xsw) {
        contentUtils.closeXml(xsw, isFullExport);
    }
    if (success) {
        Logger.info('Content assets file created successfully');
    }
    csvWriter.closeStream();
}

module.exports = {
    beforeStep: beforeStep,
    read: read,
    process: process,
    write: write,
    afterStep: afterStep
};

klevuContentAssetsExport.js

/* global empty */
/* eslint consistent-return:0 */

'use strict';

/* API Includes */
const File = require('dw/io/File');
const Site = require('dw/system/Site');
const StringUtils = require('dw/util/StringUtils');
const Calendar = require('dw/util/Calendar');

/* Script Includes*/
const CsvWriter = require('~/cartridge/scripts/utils/csvWriter');
const klevuUtils = require('~/cartridge/scripts/utils/klevuUtilsBm');
const LogUtils = require('*/cartridge/scripts/utils/klevuLogUtils');
const Logger = LogUtils.getLogger('klevuContentAssetsExport');
const contentUtils = require('~/cartridge/scripts/utils/contentAssetsUtils');

/* Global Variables*/
var xsw;
var csvWriter;
var exportModel;
var isFullExport = true;
var config = klevuUtils.config;
var writeDataResponse;
var fileClosed = false;
var cmsCount;

/**
 * beforeStep. Creation of the file and initialization XML file
 * @returns {void}
 */
function beforeStep() {
    if (isFullExport) {
        var directoryPath = StringUtils.format(config.baseKlevuPath + config.sentContentPath, Site.getCurrent().ID);
        var directory = new File(directoryPath);
        klevuUtils.deleteDirectory(directory);
    }
    csvWriter = new CsvWriter(config.sentContentHeader);
    csvWriter.initializeCSVStreamWriter(
        StringUtils.format(config.baseKlevuPath + config.sentContentPath, Site.getCurrent().ID),
        StringUtils.format(config.sentContentFileName, StringUtils.formatCalendar(new Calendar(), 'yyyyMMddHHmmssSSS'))
    );
    var ExportModel = require('~/cartridge/scripts/models/contentAssets/klevuContentAssetExportModel.js');
    exportModel = new ExportModel();
    Logger.info('Following Content Assets are getting exported: ');
}

/**
 * read. Read all the content assets
 * @returns {Array} Array of content objects
 */
function read() {
    try {
        return exportModel.getNextItem();
    } catch (exception) {
        Logger.error('ERROR :writing CMS data while reading content assets ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * process. Retrieve all the needed data for the XML file
 * @param {dw.content.Content} record - object
 * @returns {Array} Array of required fields
 */
function process(record) {
    try {
        return contentUtils.getContentFields(record);
    } catch (exception) {
        Logger.error('ERROR :writing CMS data ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * write. Write the data in the file
 * @param {Collection} lines - Actual payload for 3rd party system
 * @returns {void}
 */
function write(lines) {
    try {
        if (!empty(lines) && lines.length) {
            Logger.info('content assets count sent to write in xml : ' + lines.length);
            //xsw = contentUtils.createXml(isFullExport, config.contentObject);
            writeDataResponse = contentUtils.writeData(lines, xsw, csvWriter, isFullExport);
            //contentUtils.closeXml(xsw);
            xsw = writeDataResponse.xsw;
            fileClosed = writeDataResponse.fileClosed;
            cmsCount = writeDataResponse.cmsCount;
        }
    } catch (exception) {
        Logger.error('ERROR :writing CMS data ' + exception.stack + ' : ' + exception.message);
    }
}

/**
 * afterStep.
 * @param {boolean} success - XML creation successful or not
 * @returns {void}
 */
function afterStep(success) {
    if (fileClosed === false && xsw) {
        contentUtils.closeXml(xsw, isFullExport);
    }
    if (success) {
        Logger.info('Content assets file created successfully, count: ' + cmsCount);
    }
    csvWriter.closeStream();
}

module.exports = {
    beforeStep: beforeStep,
    read: read,
    process: process,
    write: write,
    afterStep: afterStep
};