Skip to content

Instantly share code, notes, and snippets.

@garthk
Created March 8, 2015 23:00
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save garthk/34ec2880278200b28184 to your computer and use it in GitHub Desktop.
Save garthk/34ec2880278200b28184 to your computer and use it in GitHub Desktop.
blat lines from newline-separated JSON into redis for logstash to eat
#!/usr/bin/env node
// usage:
// blat.js &
// logstash agent -f inhale-all-seen.conf &
'use strict';
var fs = require('fs'),
redis = require('redis'),
split = require('split'),
stream = require('stream');
var client = redis.createClient(),
pusher = new stream.Writable(),
count = 0;
pusher._write = function write(chunk, encoding, callback) {
chunk = chunk.toString();
if ((++count % 1000) === 0) {
process.stderr.write(count + '\r');
}
client.rpush('all-seen', chunk.trim(), callback);
};
fs.createReadStream('./all-seen.log')
.setEncoding('utf-8')
.pipe(split())
.pipe(pusher);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment