Send Rackspace Load Balancer Logs to Loggly or console output with node.js

example.md

Create a directory for this project and install the dependencies.

npm install adm-zip loggly

Download the log files for Rackspace Cloud Load Balancers from the Cloud Files container, ex: lb_123456_lb-name_Sep_2014.

Then, create an examples directory and put the zipped log files there.

Copy process-logs.js to a local file of the same name and add your loggly credentials.

Process Log Files with

node process-logs.js

You could also skip loggly all together and just use this to output logs to the console and just grep what you're looking for.

node process-logs.js | grep "search string"

process-logs.js

var fs = require('fs'),
path = require('path'),
AdmZip = require('adm-zip'),
loggly = require('loggly');

var client = loggly.createClient({
  token: "supersecret-token-here",
  subdomain: "yoursubdomain"
});

var processZippedLogs = function(dir){
  var files = fs.readdirSync(dir);
  files.forEach(function(file){
    if(file.indexOf('access_log') === 0) {
      var filePath = path.normalize(dir + '/' + file);
      console.log('processing: ' + filePath);
      unzipLog(filePath);
    }
  });
};

var unzipLog = function(filename){
  var zip = new AdmZip(filename);
  var zipEntries = zip.getEntries(); // an array of ZipEntry records

  zipEntries.forEach(function(zipEntry) {
    // only log access_log files
    if(zipEntry.entryName.indexOf('access_log') === 0){
      zip.readAsTextAsync(zipEntry, function(data, err){
        // process each line
        data.split("\n").forEach(processLine);
      });
    }
  });
};

var processLine = function(line, i, array){
  // log the line if it exists
  if(line){
    var lineInfo = line.split(' '),
    message = lineInfo.slice(2).join(' '),
    accountInfo = lineInfo[0].split('_'),
    accountID = 'CloudAccount_' + accountInfo[0],
    loadBalancerID = 'CloudLoadBalancer_' + accountInfo[1],
    domain = lineInfo[1];
    client.log(message, [accountID, loadBalancerID, domain], function(err, result){
      if (err) {
        console.log(err);
      }
      // console.log('processed line ' + i, result);
    });
    
    // uncomment the line below to log message to console
    // console.log(line)
  }
};

// ex: download rackspace cloud load balancers log files 
// from cloud files and put them in a folder called "examples" in this directory

processZippedLogs('./example');

TODO:

• This should send logs in bulk if it can, but currently the loggly nodejs library does not support it. (send logs as an array)
• One day I'll use something like pkgcloud to import the logs at an interval.

clean, son.