insert-column.js

const BSON = require('bson');
const DOC_LIMIT = 16777216;

async function insert(column) {
  // identificators for our column
  const ids = [column._id];

  const { _id, type } = column;
  // lets calculate column size with empty array
  const docMinSize = Math.max(
    // in case it will be saved in original column
    BSON.calculateObjectSize({ _id, type, values: [] }),
    // in case it will be saved in a chunk
    BSON.calculateObjectSize({ _id: uuidv4(), _next: uuidv4(), _chunk: true, values: [] })
  )

  let chunk = 0;
  let chunkStart = 0;
  let chunkSize = docMinSize;

  // we need indexes, so lets use good old `for` loop
  for (let i = 0, len = column.values.length; i < len; i++) {
    const value = column.values[i];
    size += 1; // 1byte for value type encoded
    size += String(i).length + 1; // size = key length + 1 byte for key terminator
    if (typeof value === 'string') {
      // size = 4(int32 string length) + ?(value length) + 1(value string terminator)
      size += 4 + Buffer.from(value).size + 1;
    }

    if (chunkSize > SPLIT_SIZE) {
      await saveChunk(ids, ++chunk, column.values.slice(chunkStart, i + 1));
      chunkStart = i + 1;
      chunkSize = docMinSize;
    }
  }

  if (chunk > 0) {
    // save last chunk
    await saveChunk(ids, ++chunk, column.values.slice(chunkStart), false);
    // remove values from splitted document, etc:
    column.compression = 'CHUNKS';
    column.values = [];
    column._splitted = true; // mark column as splitted
    column._next = ids[1];
  }

  // save column as is if it fits one document,
  // else column will be saved with _splitted: true, _next: firstChunkId
  return insertOneMongoDB(_id, column);
}