Skip to content

Instantly share code, notes, and snippets.

@gabrieledarrigo
Created January 12, 2022 13:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save gabrieledarrigo/31aac513c2a778b848ffd7891660e0e2 to your computer and use it in GitHub Desktop.
Save gabrieledarrigo/31aac513c2a778b848ffd7891660e0e2 to your computer and use it in GitHub Desktop.
Write JSON in a for loop, it causes a memory leak
const fs = require('fs');
const path = require('path');
const NUMBER_OF_ROWS = process.env.NUMBER_OF_ROWS || 100_000_000;
const JSON_OUTPUT = path.resolve(__dirname, `data_${Date.now()}.ndjson`);
const { info, error } = console;
(async () => {
function writeJSON() {
return new Promise((resolve, reject) => {
let index = 0;
const jsonWriter = fs.createWriteStream(JSON_OUTPUT, { flags: 'a' });
jsonWriter.on('error', reject);
jsonWriter.on('close', () => {
info(`\nCorrectly wrote ${NUMBER_OF_ROWS} in ndjson format\n`);
resolve();
});
function write(data, next) {
const wrote = jsonWriter.write(data);
if (!wrote) {
jsonWriter.once('drain', next);
} else {
process.nextTick(next);
}
}
function run() {
if (index < NUMBER_OF_ROWS) {
index++;
write(`${JSON.stringify({ data: index })}\n`, run);
} else {
jsonWriter.end();
}
}
info(`Writing ${JSON_OUTPUT} file\n`);
run();
});
}
await writeJSON();
})();
const fs = require('fs');
const path = require('path');
const NUMBER_OF_ROWS = process.env.NUMBER_OF_ROWS || 100_000_000;
const JSON_OUTPUT = path.resolve(__dirname, `data_${Date.now()}.ndjson`);
(async () => {
async function writeJSON() {
return new Promise((resolve, reject) => {
const jsonWriter = fs.createWriteStream(JSON_OUTPUT, { flags: 'a' });
jsonWriter.on('error', reject);
jsonWriter.on('close', () => {
info(`\nCorrectly wrote ${NUMBER_OF_ROWS} in ndjson format\n`);
resolve();
});
info(`Writing ${JSON_OUTPUT} file\n`);
for (let i = 0; i < NUMBER_OF_ROWS; i += 1) {
(async () => {
jsonWriter.write(`${JSON.stringify({ data: i })}\n`);
})();
}
jsonWriter.end();
});
}
await writeJSON();
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment