xml-splitter

'use strict';

const Fs = require('fs');
const Util = require('util');

const Xml2js = require('xml2js');
const XmlNodes = require('xml-nodes');
const Miss = require('mississippi');

const internals = {};

module.exports = async ({
    filePath,
    fileExtension,
    targetNode,
    rootNode = 'root',
    splitOnNumber = 400,
    nodeFilter
}) => {

    if (!filePath || !fileExtension) {
        throw new Error('Invalid arguments: filePath, fileExtension are required');
    }

    const { filterNodes } = internals;

    const fileStream = Fs.createReadStream(`${filePath}.${fileExtension}`);

    const xmlHead = '<?xml version="1.0" encoding="UTF-8"?>';

    let currentFile = '';
    let fileCount = 0;
    let currentFileTargetNodeCount = 0;

    const passThrough = (nodeAsStr, enc, next) => next(null, nodeAsStr);

    const processNode = async (nodeAsStr, enc, next) => {

        currentFile += nodeAsStr;
        currentFileTargetNodeCount++;

        if (currentFileTargetNodeCount < splitOnNumber) {
            return next();
        }

        currentFileTargetNodeCount = 0;
        const newXmlFile = `${xmlHead}\n<${rootNode}>\n${currentFile}\n</${rootNode}>`;
        currentFile = '';
        try {
            await Util.promisify(Fs.writeFile)(`${filePath}-${fileCount++}.${fileExtension}`, newXmlFile);
            next();
        }
        catch (err) {
            next(err);
        }
    };

    await Util.promisify(Miss.pipe)(
        fileStream,
        XmlNodes(targetNode),
        nodeFilter ? Miss.through(filterNodes(nodeFilter)) : Miss.through(passThrough),
        Miss.through(processNode)
    );

    // Create file for remainder
    if (currentFileTargetNodeCount > 0) {
        const newXmlFile = `${xmlHead}\n<${rootNode}>\n${currentFile}\n</${rootNode}>`;
        await Util.promisify(Fs.writeFile)(`${filePath}-${fileCount++}.${fileExtension}`, newXmlFile);
    }

    return fileCount;
};

internals.filterNodes = (filterFunc) => {

    return async (nodeAsStr, enc, next) => {

        const xmlParser = new Xml2js.Parser();

        try {
            const keep = await filterFunc(await xmlParser.parseStringPromise(nodeAsStr));
            // Let it pass on thru or not
            keep ? next(null, nodeAsStr) : next();
        }
        catch (err) {
            next(err);
        }
    };
};

werd, all makes sense. thanks for the explanations 🙏 🍷