cloudflareToSumoLogic.js

var sumoURL                = process.env.SUMO_ENDPOINT,
    zoneID                 = process.env.CLOUDFLARE_ZONE_ID,
    cloudflareAuthEmail    = process.env.CLOUDFLARE_AUTH_EMAIL,
    cloudflareAuthKey      = process.env.CLOUDFLARE_AUTH_KEY,
    sourceCategoryOverride = process.env.SOURCE_CATEGORY_OVERRIDE || 'none',
    sourceHostOverride     = process.env.SOURCE_HOST_OVERRIDE || 'api.cloudflare.com',
    sourceNameOverride     = process.env.SOURCE_NAME_OVERRIDE || zoneID;

var https = require('https');
var zlib  = require('zlib');
var url   = require('url');

function sumoMetaKey() {
  var sourceCategory = '';
  var sourceName = '';
  var sourceHost = '';

  if (sourceCategoryOverride !== null && sourceCategoryOverride !== '' && sourceCategoryOverride != 'none') {
    sourceCategory = sourceCategoryOverride;
  }

  if (sourceHostOverride !== null && sourceHostOverride !== '' && sourceHostOverride != 'none') {
    sourceHost = sourceHostOverride;
  }

  if (sourceNameOverride !== null && sourceNameOverride !== '' && sourceNameOverride != 'none') {
    sourceName = sourceNameOverride;
  }

  return sourceName + ':' + sourceCategory + ':' + sourceHost;
}

function postToSumo(context, messages) {
  var messagesTotal = Object.keys(messages).length;
  var messagesSent = 0;
  var messageErrors = [];

  var urlObject = url.parse(sumoURL);
  var options = {
    'hostname': urlObject.hostname,
    'path': urlObject.pathname,
    'method': 'POST'
  };

  var finalizeContext = function () {
    var total = messagesSent + messageErrors.length;

    if (total == messagesTotal) {
      console.log('messagesSent: ' + messagesSent + ' messagesErrors: ' + messageErrors.length);

      if (messageErrors.length > 0) {
        context.fail('errors: ' + messageErrors);
      } else {
        context.succeed();
      }
    }
  };

  Object.keys(messages).forEach(function (key, index) {
    var headerArray = key.split(':');

    options.headers = {
      'X-Sumo-Name': headerArray[0],
      'X-Sumo-Category': headerArray[1],
      'X-Sumo-Host': headerArray[2]
    };

    var req = https.request(options, function (res) {
      res.setEncoding('utf8');
      res.on('data', function (chunk) {});
      res.on('end', function () {
        if (res.statusCode == 200) {
          messagesSent++;
        } else {
          messageErrors.push('HTTP Return code ' + res.statusCode);
        }

        finalizeContext();
      });
    });

    req.on('error', function (e) {
      messageErrors.push(e.message);

      finalizeContext();
    });

    messages[key].forEach(msg => req.write(JSON.stringify(msg) + '\n'));

    req.end();
  });
}

exports.handler = function (event, context) {
  // Used to hold arrays of logs per zone I to post to SumoLogic
  var messageList = {};

  // Validate Sumo Logic URL has been set
  var urlObject = url.parse(sumoURL);
  if (urlObject.protocol != 'https:' || urlObject.host === null || urlObject.path === null) {
    context.fail('Invalid SUMO_ENDPOINT environment variable: ' + sumoURL);
  }

  // Logs are delayed by 30 minutes...
  var endTime = new Date(new Date() - (30 * 60 * 1000));

  // Set exact time
  endTime.setSeconds(0);
  endTime.setMilliseconds(0);

  var startTime = new Date(endTime - (1 * 60 * 1000));

  console.log("startTime:", startTime);
  console.log("endTime:",   endTime);

  var cloudflareOpts = {
    method:   'GET',
    hostname: 'api.cloudflare.com',
    path:     '/client/v4/zones/'+zoneID+'/logs/requests?start='+Math.floor(startTime)/1000+'&end='+Math.floor(endTime)/1000,
    headers:  {
      'X-Auth-Email': cloudflareAuthEmail,
      'X-Auth-Key': cloudflareAuthKey
    }
  };

  console.log("cloudflareOpts:", cloudflareOpts);

  var req = https.request(cloudflareOpts, function (res) {
    console.log("res.statusCode: ", res.statusCode);
    console.log("res.headers: ", res.headers);

    if (res.statusCode == 204) {
      return context.succeed();
    }

    var str = "";

    res.on("data", function (chunk) { str += chunk });
    res.on("end",  function () {
      var logs = str.split("\n");

      console.log('Log events: ' + logs.length);

      logs.forEach(function (log) {
        if (log == "") { return; }

        var parsedLog = JSON.parse(log);

        // Sumo Logic only supports 13 digit epoch time; convert original timestamp to a JSON Date object
        parsedLog.timestamp = parsedLog.timestamp/1000000;


        if (!!parsedLog.cache) {
          if (!!parsedLog.cache.startTimestamp && parsedLog.cache.startTimestamp !== null) {
            parsedLog.cache.startTimestamp = parsedLog.cache.startTimestamp/1000000
          }

          if (!!parsedLog.cache.endTimestamp && parsedLog.cache.endTimestamp !== null) {
            parsedLog.cache.endTimestamp = parsedLog.cache.endTimestamp/1000000
          }
        }

        if (!!parsedLog.edge) {
          if (!!parsedLog.edge.startTimestamp && parsedLog.edge.startTimestamp !== null) {
            parsedLog.edge.startTimestamp = parsedLog.edge.startTimestamp/1000000
          }

          if (!!parsedLog.edge.endTimestamp && parsedLog.edge.endTimestamp !== null) {
            parsedLog.edge.endTimestamp = parsedLog.edge.endTimestamp/1000000
          }
        }

        var metadataKey = sumoMetaKey();

        if (metadataKey in messageList) {
          messageList[metadataKey].push(parsedLog);
        } else {
          messageList[metadataKey] = [parsedLog];
        }
      });

      // Push messages to Sumo
      postToSumo(context, messageList);
    });
  });

  req.on("error", function (e) {
    context.fail(e);
  });

  req.end();
};