Skip to content

Instantly share code, notes, and snippets.

@DrPaulBrewer
Last active April 13, 2016 20:10
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save DrPaulBrewer/5990065 to your computer and use it in GitHub Desktop.
Save DrPaulBrewer/5990065 to your computer and use it in GitHub Desktop.
node,js asyncronous JSON.stringify stringifyToStream ----- Intended for saving JSON of a large object out to a file without using up memory or choking out other activity on the node ----- Caveat: do not modify the object as it is being written out
// Copyright 2013 Paul Brewer
// License: You may copy this file under the same terms as the MIT License
// located at http://opensource.org/licenses/MIT
// This file is provided AS IS WITH NO WARRANTY OF ANY KIND. All use is at your own risk.
//
// Status: July 13, 2013, Paul Brewer: First version. Simple test case passed. It needs to be more extensively tested.
// It lacks escaping for odd characters in strings, though a workaround is possible as described in the comments.
//
// July 14, 2013, Paul Brewer: JSON.stringify() used to stringify basic
// types such as string, number, null/undefined. This provides string
// escaping correctly, which is complex and important.
//
// .toJSON detected and called for objects like Date
//
// added indicator function for misc punctuation
// plan is now a list of items to stringify, so functions are used
// for punctuation since functions are not otherwise permitted in json
// Testing for particular punctuation is attempted with === for efficiency
// instead of evaluating the function. If an unknown function appears
// in plan, it came from the object directly, and so we ignore it.
function leftSquareBracket(){ return '[' }
function rightSquareBracket(){ return ']' }
function leftCurlyBracket(){ return '{' }
function rightCurlyBracket(){ return '}' }
function comma(){ return ',' }
function quote(){ return '"' }
function quoteColon(){ return '":' }
function colon(){ return ':' }
function stringifyToStream(obj, stream, onsuccess, onfail){
var i,l, plan=[];
function nextChunk(){
var cursor,str='';
try {
if (plan.length === 0) return onsuccess();
cursor = plan.shift();
if (typeof cursor === 'undefined'){
str = 'null'
} else if (typeof cursor === 'object'){
if (cursor === null){
str='null'
} else if (typeof cursor.toJSON === 'function'){
str = '"'+cursor.toJSON()+'"';
} else {
return stringifyToStream(cursor, stream, nextChunk, onfail)
}
} else if (typeof cursor === 'function'){
if (cursor === comma){
str = ",\n";
} else if (cursor === colon){
str = ':';
} else if (cursor === leftSquareBracket){
str = '[';
} else if (cursor === rightSquareBracket){
str = ']';
} else if (cursor === leftCurlyBracket){
str = '{';
} else if (cursor === rightCurlyBracket){
str = '}';
} else if (cursor === quote){
str = '"';
} else if (cursor === quoteColon){
str = '":';
} else return nextChunk(); // ignore unknown funcs in plan
} else str = JSON.stringify(cursor); // not a function,object,array
return stream.write(str, nextChunk);
} catch(e){
return onfail(e);
}
}
if (typeof obj !== 'object'){
return stream.write(JSON.stringify(obj), onsuccess);
}
if (typeof obj.toJSON === 'function'){
return stream.write(obj.toJSON(), onsuccess);
}
if (Object.prototype.toString.call(obj)==='[object Array]'){
plan.push(leftSquareBracket);
if (obj.length > 0){
for(i=0,l=obj.length;i<l;++i){
plan.push(obj[i], comma);
}
plan.pop(); // extra comma
}
plan.push(rightSquareBracket);
} else {
plan.push(leftCurlyBracket);
l = 0;
for (i in obj){
if (obj.hasOwnProperty(i)){
plan.push(i,colon,obj[i],comma);
++l;
}
}
if (l>0) plan.pop(); // remove last comma
plan.push(rightCurlyBracket);
}
return nextChunk();
}
exports.stringifyToStream = stringifyToStream;
var stringifyToStream = require('./stringifyToStream.js').stringifyToStream;
// simple test case
function test1(cb){
testobj = {'A': [1,3,4,6,7,8,9],
'b': { 'hi': 'there', 'ok': 78, bs: [9,8,false,'ohno!']},
'c': [ {a: 34, c: 98, e:true }, {x:56, y:true, z:false}]
};
// print out the first line with the synchronous JSON.stringify
console.log("This is test1 -- the next output is the legacy JSON.stringify, which we take as correct:");
var correct = JSON.stringify(testobj);
console.log(correct);
console.log("Check manually that it matches the following output, which is from async stringify to stream:");
function test1Compare(){
console.log();
console.log("test #1 completed -- you must judge pass/fail manually");
}
function test1Error(e){
console.log();
console.log("test #1 internal error -- "+e);
}
// after 1 sec print a seond line using the asynchronous stringifyToStream
setTimeout(
stringifyToStream.bind(null,
testobj,
process.stdout,
test1Compare,
test1Error),
1000);
}
test1();
@DrPaulBrewer
Copy link
Author

Interesting discussion on asynchronous JSON.stringify feature request for mozilla browser (rejected),

https://bugzilla.mozilla.org/show_bug.cgi?id=832664

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment