Simple JSON File Based DataStore with versioning

FileDS.js

// Import fs module
const fs = require('fs');
// Import path module
const path = require('node:path');
// Property value - used for key removal in version
const removalKey = '__DELETED_FROM_FILE__';

/**
 * Construct filename with directory
 * 
 * @param {String} directory 
 * @param {String} fn 
 * @returns Complete filename
 */
const filename = function (directory, fn) {
  return path.format({
    dir  : directory,
    base : fn
  });
}

/**
 * List all file versions for file(name)
 * 
 * @param {String} directory 
 * @param {String} name
 * @param {Number|String} version 
 * @returns Array with files
 */
const listFiles = function (directory, name, version) {
  try {
    return fs.readdirSync(directory, { withFileTypes: true })
      .filter((file) => !file.isDirectory() && new RegExp(`${name}\.*`, 'gi').test(file.name))
      .map((file) => filename(directory, file.name))
      .sort((a, b) => a.split('.').pop() - b.split('.').pop())
      .filter((file) => version ? parseInt(file.split('.').pop()) <= version : true);
  } catch (e) {
    console.error(e);
    return [];
  }
}

/**
 * Merge multiple objects into a single object
 * 
 * @returns Object
 */
const mergeObjects = function () {
  merge = function () {
    let target = arguments[0];
    for (let i = 1; i < arguments.length ; i++) {
      let arr = arguments[i];
      for (let k in arr) {
        if (Array.isArray(arr[k])) {
          if (target[k] === undefined) {            
            target[k] = [];
          }            
          target[k] = [...new Set(target[k].concat(...arr[k]))];
        } else if (typeof arr[k] === 'object') {
          if (target[k] === undefined) {            
            target[k] = {};
          }
          target[k] = merge(target[k], arr[k]);
        } else {
          if (arr[k] === removalKey) {
            delete target[k];
          } else {
            target[k] = arr[k];
          }         
        }
      }
    }
    return target;
  }
  return merge(...arguments);
}

/**
 * Merge multiple files to a single object
 * 
 * @param {Arguments} Arguments Filenames of the files to merge
 */
const mergeFiles = function () {
  return mergeObjects(...[...arguments].map((fn) => JSON.parse(fs.readFileSync(fn, 'utf8'))));
}

/**
 * Compare objects and only keep differences
 * 
 * @param {Object} original 
 * @param {Object} newVersion 
 * @returns Object with only differences between original and new version
 */
 const diffObjects = function (original, newVersion) {
  const diff = Array.isArray(newVersion) ? new Array() : Object.create({});
  // Check for deleted keys
  Object.getOwnPropertyNames(original).forEach((prop) => {
    if (typeof original[prop] === 'object') {
      diff[prop] = diffObjects(newVersion[prop], original[prop]);
    } else if(original[prop] !== newVersion[prop] && newVersion[prop] === undefined) {
      diff[prop] = removalKey;
    }
  });
  // Check for new keys and new values
  Object.getOwnPropertyNames(newVersion).forEach((prop) => {
    if (typeof newVersion[prop] === 'object') {
      diff[prop] = diffObjects(original[prop], newVersion[prop]);
      if (Array.isArray(diff[prop]) && Object.getOwnPropertyNames(diff[prop]).length === 1 || Object.getOwnPropertyNames(diff[prop]).length === 0) {
        delete diff[prop];
      }
    } else if(original[prop] !== newVersion[prop]) {
      diff[prop] = newVersion[prop];
    }
  });
  return diff;
}

/**
 * Simple file based datastore with versioning, and querying with JSONATA
 * Filenaming: <name>.<version>
 */
module.exports = class FileDS {

  // Directory where files are stored
  #directory = '';
  // Auto Merge version
  #autoMerge = false;

  // Class constructor
  constructor(options) {
    // Set directory where files are stored
    if (options && options.directory) {
      this.#directory = options.directory;
    }
    // Auto Merge
    if (options && options.autoMerge) {
      this.#autoMerge = options.autoMerge;
    }
    // If the directory does not exist, create it
    if (!fs.existsSync(this.#directory)){
        fs.mkdirSync(this.#directory, { recursive: true });
    }
  }

  /**
   * List all files and versions of this file
   * 
   * @param {String} name
   * @returns {Array}
   */
  list(name) {
    const files = listFiles(this.#directory, name);
    return files.map((file) => {
      return {
        version  : file.split('.').pop(),
        filename : file
      };
    });
  }

  /**
   * Read
   * 
   * @param {String} name
   * @param {Number|String} version 
   * @returns 
   */
  read(name, version) {
    return mergeFiles(...listFiles(this.#directory, name, version));
  }

  /**
   * Write
   * 
   * @param {String} name
   * @param {Object} data 
   * @returns Boolean
   */
  write(name, data) {
    // Write to file
    const writeFile = (name, version, data) => {
      try {
        fs.writeFileSync(filename(this.#directory, `${name}.${version}`), JSON.stringify(data));
      } catch(e) {
        console.error(e);
        return false;
      }
      return true;
    }
    // List files with this name
    const files = listFiles(this.#directory, name);
    // Check if a previous version exists
    if (files.length > 0) {
      const version = parseInt(files.map(file => file.split('.').pop()).pop()) + 1;
      // Auto Merge
      if (this.#autoMerge && version > this.#autoMerge) {
        const v1 = mergeFiles(...files);
        files.forEach((file) => fs.unlinkSync(file));
        return writeFile(name, 1, v1) && writeFile(name, 2, diffObjects(v1, data));
      }
      return writeFile(name, version, diffObjects(mergeFiles(...files), data));
    } else {
      const version = 1;
      return writeFile(name, version, data);
    }
  }

  /**
   * Get dot notation value from file
   * example: get('myfile', 'a.b.c')
   * 
   * @param {String} name 
   * @param {String} path 
   * @returns *
   */
  get(name, path) {
    const arr = (o, r, i) => !r ? o : (o[(r.slice(0, i ? -1 : r.length)).replace(/^['"]|['"]$/g, '')]);
    const dot = (o, r) => !r ? o : r.split('[').reduce(arr, o);
    return path.split('.').reduce(dot, this.read(name));
  }

  /**
   * Write dot notation value to file
   * example: set('myfile', 'a.b.c', 'some value')
   * 
   * @param {String} name 
   * @param {String} path 
   * @param {*} value 
   * @returns Boolean
   */
  set(name, path, value) {
    const data = path.split('.').reverse().reduce((p, c, i) => ({
      [c]: i === 1 && value ? {[p]: value} : p
    }));
    return this.write(name, data);
  }

  /**
   * JSONATA Query
   * 
   * @param {String} name
   * @param {String} query 
   * @returns 
   */
  query(name, query) {
    const jsonata = require('jsonata');
    return jsonata(query).evaluate(mergeFiles(...listFiles(this.#directory, name)));
  }

}