CloudWatch Events to Slack

cloudwatch-event-to-slack-lambda.md

Sends Cloudwatch Event notifications to Slack

What is this?

AWS have released a new featue called CloudWatch Events, which lets you configure events fired by cloudwatch and direct them to SNS, Lambda functions, etc. Here's the blog post

Motivational image:

Here's the motivational image:

The set of supported events sources is still somewhat limited, but this is way better than polling, which is what we had to do up until now...

In this gist you'll learn how to create such event and post updates to slack whenever an EC2 instance changes state (e.g. starting, terminating etc) You may of course use similar events to send SNS notofications to emails or anything else of that sort (in case you don't want slack)

Setup:

1. Create a slack incoming webhook and get your token
2. Encrypt this token with KMS and paste the CiphertextBlob at 'Replace with your KMS token' (KMS encryption is a bonus, not required, you may just use the token plaintext) Example how to encrypt: (replace 22e06448-f73c-42c4-b18f-74f91eb7bc1a with your own key-id from KMS - create a new key and copy the ID)

  $ aws kms encrypt --key-id 22e06448-f73c-42c4-b18f-74f91eb7bc1a --plaintext "your slack token"
  {
   "KeyId": "arn:aws:kms:us-west-2:xxxxxxxxxx:key/22e06448-f73c-42c4-b18f-74f91eb7bc1a",
   "CiphertextBlob": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"
  }

3. Create a Lambda function and replace all the const at tge header of the file with your personal data Function name: cloudwatch-event-to-slack You may use the attached test data to test this function (right after you finish the next item)
4. Give your function's role permission for the kms:Decrypt action. (assuming you're using KMS) Example:

     {
       "Version": "2012-10-17",
       "Statement": [
         {
           "Effect": "Allow",
           "Action": [
             "kms:Decrypt"
           ],
           "Resource": [
             "<your KMS key ARN>"
           ]
         }
       ]
     }

5. Add the following permission to the same IAM role as before in order to allow describing EC2 instnaces:

  {
      "Effect": "Allow",
      "Action": "ec2:Describe*",
      "Resource": "*"
  }

6. Create a new CloudWatch Event Rule to send EC2 events to this lambda function Screenshot: http://jmp.sh/uBi6dV8
7. You may test this by terminating and lanching an EC2 instance. Example output on Slack: aws.ec2 (us-west-2): i-531a188a is shutting-down
8. Prosper

cloudwatch-event-to-slack.js

//
// Sends Cloudwatch Event notifications to Slack 
//


//
// Setup: See setup in the readme file
//


const SLACK_URL = 'https://<your organization>.slack.com/services/hooks/incoming-webhook'; // Replace
const SLACK_CHANNEL_NAME = "#ops"; // Replace with yout own channel name
const kmsEncyptedToken = 'Replace with your KMS token'; // Replace
const SLACK_USERNAME = "webhookbot";

console.log('Loading function');

const AWS = require('aws-sdk');
const url = require('url');
const https = require('https');
const qs = require('querystring');


exports.handler = function(event, context) {
  console.log('event=%j', event);
  var slackReqOpts = null;
  if (slackReqOpts) {
    // Container reuse, simply process the event with the key in memory
    processEvent(event, context, slackReqOpts);
  } else {
    const encryptedBuf = new Buffer(kmsEncyptedToken, 'base64');
    const cipherText = {CiphertextBlob: encryptedBuf};

    const kms = new AWS.KMS();
    kms.decrypt(cipherText, function (err, data) {
      if (err) {
        console.log("Decrypt error: " + err);
        context.fail(err);
      } else {
        token = data.Plaintext.toString('ascii');
        const slackUrl = SLACK_URL;
        slackReqOpts = url.parse(slackUrl + token);
        slackReqOpts.method = 'POST';
        slackReqOpts.headers = {'Content-Type': 'application/json'};
        processEvent(event, context, slackReqOpts);
      }
    });
  }
};


var processEvent = function(event, context, slackReqOpts) {
  var source = event.source;
  var region = event.region;
  var text = [];
  text.push(source);
  text.push(' (' + region + ')');
  text.push(': ');

  var detail = event.detail;
  if (source === 'aws.ec2') {
    var instanceId = detail['instance-id'];
    var ec2 = new AWS.EC2();
    ec2.describeInstances({InstanceIds: [instanceId]}, function(error, data) {
      if (error) {
        console.error(error); // an error occurred
        context.fail(error);
      } else {
        var instance = data.Reservations[0].Instances[0];
        var type = instance.InstanceType
        var name = getName(instance.Tags);
        console.log('type=%s   name=%s', type, name);
        var state = detail.state;
        text.push(instanceId);
        text.push(' (')
        if (name) {
          text.push(name);
          text.push(', ');
        }
        text.push(type);
        text.push(') ')
        text.push(' is ');
        text.push(state);
        text = text.join('');
        var url = getEc2Url(instanceId, region);
        var extraText = 'See <' + url + '|' + instanceId + '>';
        postText(text, extraText, slackReqOpts, context);
      }
    });
  } else {
    text = text.join('');
    postText(text, '', slackReqOpts, context);
  }
};

var getEc2Url = function(instanceId, region) {
  return 'https://' + region + '.console.aws.amazon.com/ec2/v2/home?region=' + region + '#Instances:search=' + instanceId;
};

var postText = function(text, extraText, slackReqOpts, context) {
  var req = https.request(slackReqOpts, function (res) {
    if (res.statusCode === 200) {
      context.succeed('posted to slack');
    } else {
      context.fail('status code: ' + res.statusCode);
    }
  });

  req.on('error', function(e) {
    console.log('problem with request: ' + e.message);
    context.fail(e.message);
  });
  req.write(JSON.stringify({
    channel: SLACK_CHANNEL_NAME,
    username: SLACK_USERNAME,
    icon_emoji: ":ghost:",
    attachments: [
      {
        fallback: text,
        pretext: "AWS update",
        color: "#36a64f", // Can either be one of 'good', 'warning', 'danger', or any hex color code
        fields: [
          {
            title: text,
            value: extraText,
            short: false // Optional flag indicating whether the `value` is short enough to be displayed side-by-side with other values
          }
        ]
      }
    ]
  }));
  req.end();
};

var getName = function(tags) {
  var names = tags.filter(function(t) {
    return t.Key === 'Name';
  });
  if (names && names.length) {
    return names[0].Value;
  }
}

test-file.json

{
    "version": "0", 
    "id": "7b14d7de-3adb-4fac-9b90-6dc5019727e3", 
    "detail-type": "EC2 Instance State-change Notification", 
    "source": "aws.ec2", 
    "account": "xxxxxxxxxx", 
    "time": "2016-01-17T11:03:34Z", 
    "region": "us-west-2", 
    "resources": [ 
        "arn:aws:ec2:us-west-2:xxxxxxxxxx:instance/i-531a188a" 
    ], 
    "detail": { 
        "instance-id": "i-531a188a", 
        "state": "shutting-down" 
    } 
}

Ugh, had such high hopes for this, until I found out there's no AWS Lambda in us-west-1, so cloudwatch can't be configured to send to that. :-/ Sigh, further down the rabbithole...maybe I can ship via SQS...

In case anyone else is visiting, lambda in us-west-1 was launched Nov 21, 2016 (https://aws.amazon.com/about-aws/whats-new/2016/11/aws-lambda-available-in-us-west-n-california/)

Does this still work? When I create an Incoming Webhook in Slack I don't get a token, just a URL:
https://hooks.slack.com/services/TXXXXXX5R/XXXXXX6EA/7se4DXXXXXXXXXXXhWXXKCgWX

how can I create a lambda that recive information from CloudWatch events, change the instance Id for the instance name and then send it to sns service?