Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save ifdattic/61a5b2708dff8811e7e2 to your computer and use it in GitHub Desktop.
Save ifdattic/61a5b2708dff8811e7e2 to your computer and use it in GitHub Desktop.
Integrating DataDog & logstash on AWS EC2
Source code for article http://ifdattic.com/integrating-datadog-and-logstash-on-aws-ec2/
The first line provides the full file path, remove it from real files!
#.ebextensions/01-pre.config
commands:
100-create-posthook-dir:
command: "mkdir /opt/elasticbeanstalk/hooks/appdeploy/post"
test: "[ ! -d /opt/elasticbeanstalk/hooks/appdeploy/post ]"
container_commands:
100-chmod-posthooks:
command: "chmod +x .ebextensions/hooks/appdeploy/post/*"
200-copy-posthooks:
command: "cp .ebextensions/hooks/appdeploy/post/* /opt/elasticbeanstalk/hooks/appdeploy/post/"
#.ebextensions/02-datadog.config
packages:
yum:
datadog-agent: []
rpm:
datadog: http://yum.datadoghq.com/rpm/x86_64/datadog-agent-5.1.0-539.x86_64.rpm
files:
/etc/dd-agent/conf.d/apache.yaml:
mode: "000644"
owner: dd-agent
group: root
content: |
init_config:
instances:
- apache_status_url: http://127.0.0.1/internal/server-status?auto
container_commands:
100-copy-config:
command: "source .ebextensions/bin/copy-datadog-config.sh"
200-restart-agent:
command: "/etc/init.d/datadog-agent restart"
test: "[ $SYMFONY__ENV__DATADOG__API__KEY ]"
250-stop-agent:
command: "/etc/init.d/datadog-agent stop"
test: "[ ! $SYMFONY__ENV__DATADOG__API__KEY ]"
#.ebextensions/02-logstash.config
files:
/etc/yum.repos.d/logstash.repo:
mode: "000644"
owner: root
group: root
content: |
[logstash-1.4]
name=logstash repository for 1.4.x packages
baseurl=http://packages.elasticsearch.org/logstash/1.4/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
commands:
100-install-logstash:
command: "yum -y install logstash-1.4.2"
200-install-contrib-plugin:
command: "wget -O /tmp/logstash-contrib-1.4.2.tar.gz http://download.elasticsearch.org/logstash/logstash/logstash-contrib-1.4.2.tar.gz && tar zxf /tmp/logstash-contrib-1.4.2.tar.gz -C /opt/logstash --strip 1 && touch /tmp/logstash-contrib-installed"
test: "[ ! -f /tmp/logstash-contrib-installed ]"
container_commands:
100-delete-configs:
command: "rm -f /etc/logstash/conf.d/*"
200-copy-configs:
command: "cp .ebextensions/logstash/* /etc/logstash/conf.d/"
# 300-restart-service:
# command: "service logstash restart"
# Before rewrite to front controller in Symfony
# Allow internal requests
RewriteCond %{REQUEST_URI} !^/internal/.+$
# ---------------------------
# Enable server-status for internal IP
<Location /internal/server-status>
SetHandler server-status
Order Deny,Allow
Deny from all
Allow from 127.0.0.1
</Location>
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D %T" combined
</IfModule>
#.ebextensions/logstash/apache.conf
input {
file {
exclude => "*.gz"
path => "/var/log/httpd/*"
type => "apache-access"
}
}
filter {
grok {
match => ["message", "%{COMBINEDAPACHELOG} %{NUMBER:microseconds} %{NUMBER:seconds}"]
}
}
output {
statsd {
count => ["apache.count.bytes", "%{bytes}"]
increment => "apache.count.response.%{response}"
#increment => "apache.count.request.%{request}"
#timing => ["apache.timing.request.%{request}", "%{microseconds}"]
}
if [response] =~ /^2\d\d/ {
statsd { increment => "apache.count.response.2XX" }
} else if [response] =~ /^3\d\d/ {
statsd { increment => "apache.count.response.3XX" }
} else if [response] =~ /^4\d\d/ {
statsd { increment => "apache.count.response.4XX" }
} else if [response] =~ /^5\d\d/ {
statsd { increment => "apache.count.response.5XX" }
}
}
#.ebextensions/bin/copy-datadog-config.sh
#!/bin/bash
sed 's/api_key:.*/api_key: '"$SYMFONY__ENV__DATADOG__API__KEY"'/' /etc/dd-agent/datadog.conf.example > /etc/dd-agent/datadog.conf
#.ebextensions/hooks/appdeploy/post/logstash-background-job.sh
#!/bin/bash
. /opt/elasticbeanstalk/support/envvars
for pid in `ps aux | grep /etc/logstash/conf.d | grep -v grep | tr -s ' ' | cut -d ' ' -f 2`
do
disown $pid
kill -9 $pid
done
export HOME=/var/lib/logstash
/opt/logstash/bin/logstash \
-f /etc/logstash/conf.d \
-l /var/log/logstash/logstash.log \
> /var/log/logstash/logstash.stdout \
2> /var/log/logstash/logstash.err \
&
@ifdattic
Copy link
Author

First line is the full location, remove from files

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment