Skip to content

Instantly share code, notes, and snippets.

@stephenplusplus
Last active December 7, 2020 15:02
Show Gist options
  • Save stephenplusplus/62f747a4329f5dc1a461f93bb14fbf3d to your computer and use it in GitHub Desktop.
Save stephenplusplus/62f747a4329f5dc1a461f93bb14fbf3d to your computer and use it in GitHub Desktop.
logging-issue-943
const assert = require('assert');
const Logging = require('@google-cloud/logging').Logging;
const v4 = require('uuid').v4;
const TESTS_PREFIX = 'issue-943';
const TIME_TEST_STARTED = new Date().toISOString();
const NUM_LOGS_TO_CREATE = 50;
const NUM_ENTRIES_TO_WRITE = 100000;
const FLUSH_AFTER_NUM_LOGS_FILLED = NUM_LOGS_TO_CREATE / 10;
const NUM_WRITES_TO_MAKE = 50;
const WRITE_CONSISTENCY_DELAY_MS = 10000;
const logging = new Logging();
function generateName() {
return `${TESTS_PREFIX}-${Date.now()}-${v4().split('-').pop()}`;
}
run();
async function run() {
const logNamesCreated = [];
function validateEntriesFromLogCallback(log, callback) {
let numEntriesReceived = 0;
setTimeout(pollForMessages, WRITE_CONSISTENCY_DELAY_MS, {
autoPaginate: false,
filter: `timestamp > "${TIME_TEST_STARTED}"`,
pageSize: 1000,
});
function pollForMessages(query) {
log.getEntries(query, (err, entries, nextQuery) => {
if (err) {
callback(err);
return;
}
assert(entries.every(e => e.data.originLog === log.name));
numEntriesReceived += entries.length;
console.log(`${numEntriesReceived}/${NUM_ENTRIES_TO_WRITE} entries received`);
if (nextQuery) {
setTimeout(pollForMessages, WRITE_CONSISTENCY_DELAY_MS, nextQuery);
return;
} else {
if (numEntriesReceived < NUM_ENTRIES_TO_WRITE) {
callback(new Error(`Not enough entries received`));
return;
}
}
callback();
});
}
}
function validateEntriesFromLog(log) {
return new Promise((res, rej) => {
validateEntriesFromLogCallback(log, (err, entries) => {
if (err) {
rej(err);
return;
}
res(entries);
});
});
}
const logAndEntrySets = [];
let numLogsCreated = 0;
while (numLogsCreated++ < NUM_LOGS_TO_CREATE) {
const logName = generateName();
logNamesCreated.push(logName);
const log = logging.log(logName);
log.shortName = `log #${numLogsCreated}`;
logAndEntrySets.push({log, entries: []});
}
let numLogsFilled = 0;
while (numLogsFilled < NUM_LOGS_TO_CREATE) {
const logAndEntrySetBatch = logAndEntrySets.splice(0, FLUSH_AFTER_NUM_LOGS_FILLED);
for (const {log, entries} of logAndEntrySetBatch) {
let numEntriesWritten = 0;
while (numEntriesWritten++ < NUM_ENTRIES_TO_WRITE) {
entries.push(log.entry({originLog: log.name}));
}
}
console.log('Flushing logs...');
await writeLogs(logAndEntrySetBatch);
numLogsFilled += logAndEntrySetBatch.length;
}
async function writeLogs(logAndEntrySets) {
const numEntriesPerWrite = NUM_ENTRIES_TO_WRITE / NUM_WRITES_TO_MAKE;
let numWrites = 0;
while (numWrites < NUM_WRITES_TO_MAKE) {
const numEntriesWritten = numWrites * numEntriesPerWrite;
for (const {log, entries} of logAndEntrySets) {
console.log(
`${log.shortName} writing ${numEntriesWritten}-${
numEntriesWritten + numEntriesPerWrite
}/${NUM_ENTRIES_TO_WRITE} entries`
);
await log.write(entries.splice(0, numEntriesPerWrite));
}
numWrites++;
}
}
console.log('Validating logs...');
let logNamesProcessed = 0;
for (const logName of logNamesCreated) {
console.log(`Validating log #${++logNamesProcessed}: ${logName}`);
const log = logging.log(logName);
await validateEntriesFromLog(log);
}
}
{
"name": "@google-cloud/logging",
"version": "1.0.0",
"scripts": {
"test": "node ./logging-issue-943.js"
},
"dependencies": {
"@google-cloud/logging": "^9.0.0",
"uuid": "^8.0.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment