Created

Embed URL

HTTPS clone URL

SSH clone URL

You can clone with HTTPS or SSH.

Download Gist

My attempt at the minimal code to read text files one line at a time in node.js

View linebyline.js
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
var fs = require('fs'),
StringDecoder = require('string_decoder').StringDecoder,
util = require('util');
 
function lineByLine(fd) {
var blob = '';
var blobStart = 0;
var blobEnd = 0;
 
var decoder = new StringDecoder('utf8');
var CHUNK_SIZE = 16384;
var chunk = new Buffer(CHUNK_SIZE);
 
var eolPos = -1;
var lastChunk = false;
 
var moreLines = true;
var readMore = true;
 
// each line
while (moreLines) {
 
readMore = true;
// append more chunks from the file onto the end of our blob of text until we have an EOL or EOF
while (readMore) {
 
// do we have a whole line? (with LF)
eolPos = blob.indexOf('\n', blobStart);
 
if (eolPos !== -1) {
blobEnd = eolPos;
readMore = false;
 
// do we have the last line? (no LF)
} else if (lastChunk) {
blobEnd = blob.length;
readMore = false;
 
// otherwise read more
} else {
var bytesRead = fs.readSync(fd, chunk, 0, CHUNK_SIZE, null);
 
lastChunk = bytesRead !== CHUNK_SIZE;
 
// TODO blob grows until memory is exhausted!
blob += decoder.write(chunk.slice(0, bytesRead));
}
}
 
if (blobStart < blob.length) {
processLine(blob.substring(blobStart, blobEnd + 1));
 
blobStart = blobEnd + 1;
 
if (blobStart >= CHUNK_SIZE) {
// blobStart is in characters, CHUNK_SIZE is in octets
var freeable = blobStart / CHUNK_SIZE;
 
blob = blob.substring(CHUNK_SIZE);
blobStart -= CHUNK_SIZE;
blobEnd -= CHUNK_SIZE;
}
} else {
moreLines = false;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.