Instantly share code, notes, and snippets.

@eddywashere eddywashere/example.md
Last active Aug 29, 2015

Embed
What would you like to do?
Send Rackspace Load Balancer Logs to Loggly or console output with node.js

Create a directory for this project and install the dependencies.

npm install adm-zip loggly

Download the log files for Rackspace Cloud Load Balancers from the Cloud Files container, ex: lb_123456_lb-name_Sep_2014.

Then, create an examples directory and put the zipped log files there.

Copy process-logs.js to a local file of the same name and add your loggly credentials.

Process Log Files with

node process-logs.js

You could also skip loggly all together and just use this to output logs to the console and just grep what you're looking for.

node process-logs.js | grep "search string"
var fs = require('fs'),
path = require('path'),
AdmZip = require('adm-zip'),
loggly = require('loggly');
var client = loggly.createClient({
token: "supersecret-token-here",
subdomain: "yoursubdomain"
});
var processZippedLogs = function(dir){
var files = fs.readdirSync(dir);
files.forEach(function(file){
if(file.indexOf('access_log') === 0) {
var filePath = path.normalize(dir + '/' + file);
console.log('processing: ' + filePath);
unzipLog(filePath);
}
});
};
var unzipLog = function(filename){
var zip = new AdmZip(filename);
var zipEntries = zip.getEntries(); // an array of ZipEntry records
zipEntries.forEach(function(zipEntry) {
// only log access_log files
if(zipEntry.entryName.indexOf('access_log') === 0){
zip.readAsTextAsync(zipEntry, function(data, err){
// process each line
data.split("\n").forEach(processLine);
});
}
});
};
var processLine = function(line, i, array){
// log the line if it exists
if(line){
var lineInfo = line.split(' '),
message = lineInfo.slice(2).join(' '),
accountInfo = lineInfo[0].split('_'),
accountID = 'CloudAccount_' + accountInfo[0],
loadBalancerID = 'CloudLoadBalancer_' + accountInfo[1],
domain = lineInfo[1];
client.log(message, [accountID, loadBalancerID, domain], function(err, result){
if (err) {
console.log(err);
}
// console.log('processed line ' + i, result);
});
// uncomment the line below to log message to console
// console.log(line)
}
};
// ex: download rackspace cloud load balancers log files
// from cloud files and put them in a folder called "examples" in this directory
processZippedLogs('./example');
@eddywashere

This comment has been minimized.

Copy link
Owner Author

eddywashere commented Sep 25, 2014

TODO:

  • This should send logs in bulk if it can, but currently the loggly nodejs library does not support it. (send logs as an array)
  • One day I'll use something like pkgcloud to import the logs at an interval.
@cbfx

This comment has been minimized.

Copy link

cbfx commented Sep 25, 2014

clean, son.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment