logging-issue-943

logging-issue-943.js

const assert = require('assert');
const Logging = require('@google-cloud/logging').Logging;
const v4 = require('uuid').v4;

const TESTS_PREFIX = 'issue-943';
const TIME_TEST_STARTED = new Date().toISOString();

const NUM_LOGS_TO_CREATE = 50;
const NUM_ENTRIES_TO_WRITE = 100000;
const FLUSH_AFTER_NUM_LOGS_FILLED = NUM_LOGS_TO_CREATE / 10;
const NUM_WRITES_TO_MAKE = 50;
const WRITE_CONSISTENCY_DELAY_MS = 10000;

const logging = new Logging();

function generateName() {
  return `${TESTS_PREFIX}-${Date.now()}-${v4().split('-').pop()}`;
}

run();

async function run() {
  const logNamesCreated = [];

  function validateEntriesFromLogCallback(log, callback) {
    let numEntriesReceived = 0;

    setTimeout(pollForMessages, WRITE_CONSISTENCY_DELAY_MS, {
      autoPaginate: false,
      filter: `timestamp > "${TIME_TEST_STARTED}"`,
      pageSize: 1000,
    });

    function pollForMessages(query) {
      log.getEntries(query, (err, entries, nextQuery) => {
        if (err) {
          callback(err);
          return;
        }

        assert(entries.every(e => e.data.originLog === log.name));

        numEntriesReceived += entries.length;
        console.log(`${numEntriesReceived}/${NUM_ENTRIES_TO_WRITE} entries received`);

        if (nextQuery) {
          setTimeout(pollForMessages, WRITE_CONSISTENCY_DELAY_MS, nextQuery);
          return;
        } else {
          if (numEntriesReceived < NUM_ENTRIES_TO_WRITE) {
            callback(new Error(`Not enough entries received`));
            return;
          }
        }

        callback();
      });
    }
  }

  function validateEntriesFromLog(log) {
    return new Promise((res, rej) => {
      validateEntriesFromLogCallback(log, (err, entries) => {
        if (err) {
          rej(err);
          return;
        }
        res(entries);
      });
    });
  }

  const logAndEntrySets = [];

  let numLogsCreated = 0;
  while (numLogsCreated++ < NUM_LOGS_TO_CREATE) {
    const logName = generateName();
    logNamesCreated.push(logName);
    const log = logging.log(logName);
    log.shortName = `log #${numLogsCreated}`;
    logAndEntrySets.push({log, entries: []});
  }

  let numLogsFilled = 0;
  while (numLogsFilled < NUM_LOGS_TO_CREATE) {
    const logAndEntrySetBatch = logAndEntrySets.splice(0, FLUSH_AFTER_NUM_LOGS_FILLED);

    for (const {log, entries} of logAndEntrySetBatch) {
      let numEntriesWritten = 0;
      while (numEntriesWritten++ < NUM_ENTRIES_TO_WRITE) {
        entries.push(log.entry({originLog: log.name}));
      }
    }

    console.log('Flushing logs...');
    await writeLogs(logAndEntrySetBatch);
    numLogsFilled += logAndEntrySetBatch.length;
  }

  async function writeLogs(logAndEntrySets) {
    const numEntriesPerWrite = NUM_ENTRIES_TO_WRITE / NUM_WRITES_TO_MAKE;
    let numWrites = 0;

    while (numWrites < NUM_WRITES_TO_MAKE) {
      const numEntriesWritten = numWrites * numEntriesPerWrite;
      for (const {log, entries} of logAndEntrySets) {
        console.log(
          `${log.shortName} writing ${numEntriesWritten}-${
            numEntriesWritten + numEntriesPerWrite
          }/${NUM_ENTRIES_TO_WRITE} entries`
        );
        await log.write(entries.splice(0, numEntriesPerWrite));
      }
      numWrites++;
    }
  }

  console.log('Validating logs...');
  let logNamesProcessed = 0;
  for (const logName of logNamesCreated) {
    console.log(`Validating log #${++logNamesProcessed}: ${logName}`);
    const log = logging.log(logName);
    await validateEntriesFromLog(log);
  }
}

package.json

{
  "name": "@google-cloud/logging",
  "version": "1.0.0",
  "scripts": {
    "test": "node ./logging-issue-943.js"
  },
  "dependencies": {
    "@google-cloud/logging": "^9.0.0",
    "uuid": "^8.0.0"
  }
}