Integrating DataDog & logstash on AWS EC2

00-info-integrating-datadog-and-logstash-on-aws-ec2

Source code for article http://ifdattic.com/integrating-datadog-and-logstash-on-aws-ec2/

The first line provides the full file path, remove it from real files!

01-pre.config

#.ebextensions/01-pre.config
commands:
    100-create-posthook-dir:
        command: "mkdir /opt/elasticbeanstalk/hooks/appdeploy/post"
        test: "[ ! -d /opt/elasticbeanstalk/hooks/appdeploy/post ]"
container_commands:
    100-chmod-posthooks:
        command: "chmod +x .ebextensions/hooks/appdeploy/post/*"
    200-copy-posthooks:
        command: "cp .ebextensions/hooks/appdeploy/post/* /opt/elasticbeanstalk/hooks/appdeploy/post/"

02-datadog.config

#.ebextensions/02-datadog.config
packages:
    yum:
        datadog-agent: []
    rpm:
        datadog: http://yum.datadoghq.com/rpm/x86_64/datadog-agent-5.1.0-539.x86_64.rpm

files:
    /etc/dd-agent/conf.d/apache.yaml:
        mode: "000644"
        owner: dd-agent
        group: root
        content: |
            init_config:

            instances:
                -   apache_status_url: http://127.0.0.1/internal/server-status?auto

container_commands:
    100-copy-config:
        command: "source .ebextensions/bin/copy-datadog-config.sh"
    200-restart-agent:
        command: "/etc/init.d/datadog-agent restart"
        test: "[ $SYMFONY__ENV__DATADOG__API__KEY ]"
    250-stop-agent:
        command: "/etc/init.d/datadog-agent stop"
        test: "[ ! $SYMFONY__ENV__DATADOG__API__KEY ]"

02-logstash.config

#.ebextensions/02-logstash.config
files:
    /etc/yum.repos.d/logstash.repo:
        mode: "000644"
        owner: root
        group: root
        content: |
            [logstash-1.4]
            name=logstash repository for 1.4.x packages
            baseurl=http://packages.elasticsearch.org/logstash/1.4/centos
            gpgcheck=1
            gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
            enabled=1

commands:
    100-install-logstash:
        command: "yum -y install logstash-1.4.2"
    200-install-contrib-plugin:
        command: "wget -O /tmp/logstash-contrib-1.4.2.tar.gz http://download.elasticsearch.org/logstash/logstash/logstash-contrib-1.4.2.tar.gz && tar zxf /tmp/logstash-contrib-1.4.2.tar.gz -C /opt/logstash --strip 1 && touch /tmp/logstash-contrib-installed"
        test: "[ ! -f /tmp/logstash-contrib-installed ]"

container_commands:
    100-delete-configs:
        command: "rm -f /etc/logstash/conf.d/*"
    200-copy-configs:
        command: "cp .ebextensions/logstash/* /etc/logstash/conf.d/"
    # 300-restart-service:
    #     command: "service logstash restart"

Apache

# Before rewrite to front controller in Symfony
# Allow internal requests
RewriteCond %{REQUEST_URI} !^/internal/.+$

# ---------------------------

# Enable server-status for internal IP
<Location /internal/server-status>
   SetHandler server-status
   Order Deny,Allow
   Deny from all
   Allow from 127.0.0.1
</Location>

<IfModule log_config_module>
    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D %T" combined
</IfModule>

apache.conf

#.ebextensions/logstash/apache.conf
input {
  file {
    exclude => "*.gz"
    path => "/var/log/httpd/*"
    type => "apache-access"
  }
}

filter {
  grok {
    match => ["message", "%{COMBINEDAPACHELOG} %{NUMBER:microseconds} %{NUMBER:seconds}"]
  }
}

output {
  statsd {
    count => ["apache.count.bytes", "%{bytes}"]
    increment => "apache.count.response.%{response}"
    #increment => "apache.count.request.%{request}"
    #timing => ["apache.timing.request.%{request}", "%{microseconds}"]
  }

  if [response] =~ /^2\d\d/ {
    statsd { increment => "apache.count.response.2XX" }
  } else if [response] =~ /^3\d\d/ {
    statsd { increment => "apache.count.response.3XX" }
  } else if [response] =~ /^4\d\d/ {
    statsd { increment => "apache.count.response.4XX" }
  } else if [response] =~ /^5\d\d/ {
    statsd { increment => "apache.count.response.5XX" }
  }
}

copy-datadog-config.sh

#.ebextensions/bin/copy-datadog-config.sh
#!/bin/bash

sed 's/api_key:.*/api_key: '"$SYMFONY__ENV__DATADOG__API__KEY"'/' /etc/dd-agent/datadog.conf.example > /etc/dd-agent/datadog.conf

logstash-background-job.sh

#.ebextensions/hooks/appdeploy/post/logstash-background-job.sh
#!/bin/bash

. /opt/elasticbeanstalk/support/envvars

for pid in `ps aux | grep /etc/logstash/conf.d | grep -v grep | tr -s ' ' | cut -d ' ' -f 2`
do
    disown $pid
    kill -9 $pid
done

export HOME=/var/lib/logstash

/opt/logstash/bin/logstash \
    -f /etc/logstash/conf.d \
    -l /var/log/logstash/logstash.log \
    > /var/log/logstash/logstash.stdout \
    2> /var/log/logstash/logstash.err \
    &