Last active
September 8, 2019 13:42
-
-
Save peey/e35b5dc3702c82dfdb5bc914f54253c3 to your computer and use it in GitHub Desktop.
Writes large files with low memory requirements
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var items = 2500000000; // number of lines to write to the file. Currently >20 GB for sequential ints | |
var buffer_size = 1000000; // amount of items to buffer in memory. Currently ~200 MB for ints | |
var fs = require('fs') | |
var buffer = []; | |
for (var i =0; i < items; i++) { | |
var data = i; // make it a random integer, or whatever else your heart desires | |
buffer.push(data); | |
if (buffer.length == ) { | |
fs.appendFileSync('large-file', buffer.join("\n")) | |
buffer = []; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment