Skip to content

Instantly share code, notes, and snippets.

@kevzettler
Created July 21, 2014 23:38
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kevzettler/d124ca0ade48d8353555 to your computer and use it in GitHub Desktop.
Save kevzettler/d124ca0ade48d8353555 to your computer and use it in GitHub Desktop.
nodejs file reads
var q = async.queue(function (file, cb) {
console.log("from the queue::", file.path);
var stream = fs.createReadStream(file.path)
// ...
stream.on('end', function() {
// finish up, then
cb();
});
}, 2);
var globStream = gs.create(getGlobPaths());
globStream.on('data', function(file) {
console.log("globStream::", file.path);
q.push(file);
});
globStream.on('end', function() {
// We don't want to add the `drain` handler until *after* the globstream
// finishes. Otherwise, we could end up in a situation where the globber
// is still running but all pending file read operations have finished.
q.drain = function() {
// All done with everything.
};
// ...and if the queue is empty when the globber finishes, make sure the done
// callback gets called.
if (q.idle()) q.drain();
});
@kevzettler
Copy link
Author

globStream:: /events/kev/2014-07-18/0-0/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-0/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-0/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-0/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-0/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-0/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-1/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-2/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-3/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-4/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-5/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-6/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/0-7/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-0/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-1/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-2/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-3/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-4/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-5/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-6/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/1-7/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-0/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-1/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-2/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-3/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-4/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-5/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-6/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/2-7/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-0/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-1/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-2/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-3/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-4/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-5/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-6/21:20:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/19:45:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/19:50:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/20:05:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/20:10:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/20:15:00.csv.gz
globStream:: /events/kev/2014-07-18/3-7/21:20:00.csv.gz
from the queu:: /events/kev/2014-07-18/0-0/19:45:00.csv.gz
from the queu:: /events/kev/2014-07-18/0-0/19:50:00.csv.gz

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment