Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Integrating DataDog & logstash on AWS EC2
Source code for article http://ifdattic.com/integrating-datadog-and-logstash-on-aws-ec2/
The first line provides the full file path, remove it from real files!
#.ebextensions/01-pre.config
commands:
100-create-posthook-dir:
command: "mkdir /opt/elasticbeanstalk/hooks/appdeploy/post"
test: "[ ! -d /opt/elasticbeanstalk/hooks/appdeploy/post ]"
container_commands:
100-chmod-posthooks:
command: "chmod +x .ebextensions/hooks/appdeploy/post/*"
200-copy-posthooks:
command: "cp .ebextensions/hooks/appdeploy/post/* /opt/elasticbeanstalk/hooks/appdeploy/post/"
#.ebextensions/02-datadog.config
packages:
yum:
datadog-agent: []
rpm:
datadog: http://yum.datadoghq.com/rpm/x86_64/datadog-agent-5.1.0-539.x86_64.rpm
files:
/etc/dd-agent/conf.d/apache.yaml:
mode: "000644"
owner: dd-agent
group: root
content: |
init_config:
instances:
- apache_status_url: http://127.0.0.1/internal/server-status?auto
container_commands:
100-copy-config:
command: "source .ebextensions/bin/copy-datadog-config.sh"
200-restart-agent:
command: "/etc/init.d/datadog-agent restart"
test: "[ $SYMFONY__ENV__DATADOG__API__KEY ]"
250-stop-agent:
command: "/etc/init.d/datadog-agent stop"
test: "[ ! $SYMFONY__ENV__DATADOG__API__KEY ]"
#.ebextensions/02-logstash.config
files:
/etc/yum.repos.d/logstash.repo:
mode: "000644"
owner: root
group: root
content: |
[logstash-1.4]
name=logstash repository for 1.4.x packages
baseurl=http://packages.elasticsearch.org/logstash/1.4/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
commands:
100-install-logstash:
command: "yum -y install logstash-1.4.2"
200-install-contrib-plugin:
command: "wget -O /tmp/logstash-contrib-1.4.2.tar.gz http://download.elasticsearch.org/logstash/logstash/logstash-contrib-1.4.2.tar.gz && tar zxf /tmp/logstash-contrib-1.4.2.tar.gz -C /opt/logstash --strip 1 && touch /tmp/logstash-contrib-installed"
test: "[ ! -f /tmp/logstash-contrib-installed ]"
container_commands:
100-delete-configs:
command: "rm -f /etc/logstash/conf.d/*"
200-copy-configs:
command: "cp .ebextensions/logstash/* /etc/logstash/conf.d/"
# 300-restart-service:
# command: "service logstash restart"
# Before rewrite to front controller in Symfony
# Allow internal requests
RewriteCond %{REQUEST_URI} !^/internal/.+$
# ---------------------------
# Enable server-status for internal IP
<Location /internal/server-status>
SetHandler server-status
Order Deny,Allow
Deny from all
Allow from 127.0.0.1
</Location>
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D %T" combined
</IfModule>
#.ebextensions/logstash/apache.conf
input {
file {
exclude => "*.gz"
path => "/var/log/httpd/*"
type => "apache-access"
}
}
filter {
grok {
match => ["message", "%{COMBINEDAPACHELOG} %{NUMBER:microseconds} %{NUMBER:seconds}"]
}
}
output {
statsd {
count => ["apache.count.bytes", "%{bytes}"]
increment => "apache.count.response.%{response}"
#increment => "apache.count.request.%{request}"
#timing => ["apache.timing.request.%{request}", "%{microseconds}"]
}
if [response] =~ /^2\d\d/ {
statsd { increment => "apache.count.response.2XX" }
} else if [response] =~ /^3\d\d/ {
statsd { increment => "apache.count.response.3XX" }
} else if [response] =~ /^4\d\d/ {
statsd { increment => "apache.count.response.4XX" }
} else if [response] =~ /^5\d\d/ {
statsd { increment => "apache.count.response.5XX" }
}
}
#.ebextensions/bin/copy-datadog-config.sh
#!/bin/bash
sed 's/api_key:.*/api_key: '"$SYMFONY__ENV__DATADOG__API__KEY"'/' /etc/dd-agent/datadog.conf.example > /etc/dd-agent/datadog.conf
#.ebextensions/hooks/appdeploy/post/logstash-background-job.sh
#!/bin/bash
. /opt/elasticbeanstalk/support/envvars
for pid in `ps aux | grep /etc/logstash/conf.d | grep -v grep | tr -s ' ' | cut -d ' ' -f 2`
do
disown $pid
kill -9 $pid
done
export HOME=/var/lib/logstash
/opt/logstash/bin/logstash \
-f /etc/logstash/conf.d \
-l /var/log/logstash/logstash.log \
> /var/log/logstash/logstash.stdout \
2> /var/log/logstash/logstash.err \
&
@ifdattic

This comment has been minimized.

Copy link
Owner Author

@ifdattic ifdattic commented Nov 26, 2014

First line is the full location, remove from files

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.