Writes large files with low memory requirements
var items = 2500000000; // number of lines to write to the file. Currently >20 GB for sequential ints | |
var buffer_size = 1000000; // amount of items to buffer in memory. Currently ~200 MB for ints | |
var fs = require('fs') | |
var buffer = []; | |
for (var i =0; i < items; i++) { | |
var data = i; // make it a random integer, or whatever else your heart desires | |
buffer.push(data); | |
if (buffer.length == ) { | |
fs.appendFileSync('large-file', buffer.join("\n")) | |
buffer = []; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment