public
Created

My attempt at the minimal code to read text files one line at a time in node.js

  • Download Gist
linebyline.js
JavaScript
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
var fs = require('fs'),
StringDecoder = require('string_decoder').StringDecoder,
util = require('util');
 
function lineByLine(fd) {
var blob = '';
var blobStart = 0;
var blobEnd = 0;
 
var decoder = new StringDecoder('utf8');
var CHUNK_SIZE = 16384;
var chunk = new Buffer(CHUNK_SIZE);
 
var eolPos = -1;
var lastChunk = false;
 
var moreLines = true;
var readMore = true;
 
// each line
while (moreLines) {
 
readMore = true;
// append more chunks from the file onto the end of our blob of text until we have an EOL or EOF
while (readMore) {
 
// do we have a whole line? (with LF)
eolPos = blob.indexOf('\n', blobStart);
 
if (eolPos !== -1) {
blobEnd = eolPos;
readMore = false;
 
// do we have the last line? (no LF)
} else if (lastChunk) {
blobEnd = blob.length;
readMore = false;
 
// otherwise read more
} else {
var bytesRead = fs.readSync(fd, chunk, 0, CHUNK_SIZE, null);
 
lastChunk = bytesRead !== CHUNK_SIZE;
 
// TODO blob grows until memory is exhausted!
blob += decoder.write(chunk.slice(0, bytesRead));
}
}
 
if (blobStart < blob.length) {
processLine(blob.substring(blobStart, blobEnd + 1));
 
blobStart = blobEnd + 1;
 
if (blobStart >= CHUNK_SIZE) {
// blobStart is in characters, CHUNK_SIZE is in octets
var freeable = blobStart / CHUNK_SIZE;
 
blob = blob.substring(CHUNK_SIZE);
blobStart -= CHUNK_SIZE;
blobEnd -= CHUNK_SIZE;
}
} else {
moreLines = false;
}
}
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.