Scan from many files, opening only one at a time.

json.gz.scanner.with.EventEmitter.js

var scangz = function (fname) {
    var rawread = fs.createReadStream(fname);
    var unzipped = rawread.pipe(zlib.createUnzip());
    return unzipped;
}

var FileScanner = function() {
    var self = this;
    events.EventEmitter.call(self);

    self.results = new Readable({objectMode: true});
    self.results._read = function noop() {};

    self.on('getFiles', function(files, filter) {
         file = files.shift();
         var scanner = scangz(file);
         var scannerp = scanner.pipe(jsonstream());
         console.log("\nopened", file);
         scannerp.on('data', function(chunk) {
             try {
                 if (filter(chunk)) {
                     self.results.push(chunk);
                 }
             } catch (ev) {
                 console.log("ERROR: ", ev);
                 console.log(ev.stack);
             }
         });

         scannerp.on('end', function() {
             console.log("END OF STREAM");
             if (files.length > 0) {
                 self.emit('getFiles', files, filter);
             }
         });
    });
}
util.inherits(FileScanner, events.EventEmitter);


// Return a stream
module.exports.scanner = function() {

    var files = ...get list of files...

    var filescanner = new FileScanner();
    filescanner.emit('getFiles', files, filter);

    return filescanner.results;
}