Last active
December 14, 2015 10:10
-
-
Save steve-taylor/5070387 to your computer and use it in GitHub Desktop.
Streaming string tokenizer, made specifically for Node.js, but could be used in the browser too. Note fully tested yet!
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function StreamingTokenizer(delimiter, callback) { | |
this.delimiter = delimiter; | |
this.callback = callback; | |
this.pendingText = ''; | |
this.open = true; | |
} | |
StreamingTokenizer.prototype = { | |
add: function(text) { | |
if (!this.open) { | |
throw 'StreamingTokenizer closed'; | |
} | |
this.pendingText += text; | |
var lines = this.pendingText.split(this.delimiter); | |
if (lines.length == 0) { | |
return; | |
} | |
for (var i = 0; i < lines.length - 1; ++i) { | |
this.callback(lines[i]); | |
} | |
this.pendingText = lines[lines.length - 1]; | |
}, | |
close: function() { | |
if (this.pendingText.length > 0) { | |
this.callback(this.pendingText); | |
} | |
this.pendingText = ''; | |
this.open = false; | |
} | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment