Skip to content

Instantly share code, notes, and snippets.

@stevepereira
Forked from fxtentacle/agent.conf
Created July 25, 2013 18:20
Show Gist options
  • Save stevepereira/6082363 to your computer and use it in GitHub Desktop.
Save stevepereira/6082363 to your computer and use it in GitHub Desktop.
input {
exec {
type => "dstat"
command => "dstat -cdngypms --nocolor 1 0"
interval => 13
}
exec {
type => "apache-benchmark"
command => "ab -t 5 -n 1 -c 1 -d -S 'https://spratshop-rails.spratshop.com/' | grep -E '(requests:|Time taken)'"
interval => 17
}
file {
type => "linux-syslog"
path => [
"/var/log/alternatives.log",
"/var/log/auth.log",
"/var/log/daemon.log",
"/var/log/dpkg.log",
"/var/log/faillog",
"/var/log/fontconfig.log",
"/var/log/kern.log",
"/var/log/lastlog",
"/var/log/lpr.log",
"/var/log/mail.log",
"/var/log/mysql.log",
"/var/log/pycentral.log",
# "/var/log/rabbitmq_logger.log",
"/var/log/user.log",
"/var/log/messages",
"/var/log/syslog"
]
}
file {
type => "apache-access"
path => [
"/var/log/apache2/access.log",
"/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-access.log",
"/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-access.log",
"/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-access.log",
"/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-access.log",
"/var/log/apache2/other_vhosts_access.log"
]
}
file {
type => "apache-rewrite"
path => [
"/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-rewrite.log",
"/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-rewrite.log",
"/var/log/apache2/rewrite.log"
]
}
file {
type => "apache-error"
path => [
"/var/log/apache2/error.log",
"/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-error.log",
"/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-error.log",
"/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-error.log",
"/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-error.log"
]
}
file {
type => "rabbitmq-post-bridge"
path => [
"/var/log/rabbitmq_post_bridge.log"
]
}
file {
type => "spratshop-rails"
path => [
"/opt/spratshop/rails/shared/log/production.log",
"/opt/spratshop/rails/shared/log/development.log"
]
}
file {
type => "spratshop-file-sender"
path => [
"/var/log/spratshop_file_sender.log"
]
}
file {
type => "spratshop-ftp-retry-cron"
path => [
"/var/log/spratshop_ftp_retry_cron.log"
]
}
file {
type => "spratshop-status-checker"
path => [
"/var/log/spratshop_status_checker.log"
]
}
file {
type => "spratshop-usage-cron"
path => [
"/var/log/spratshop_usage_cron.log"
]
}
file {
type => "vsftpd"
path => [
"/var/log/vsftpd.log"
]
}
file {
type => "vsftpd-to-rabbitmq"
path => [
"/var/log/vsftpd_to_rabbitmq.log"
]
}
file {
type => "spratshop-image-processor"
path => [
"/var/log/spratshop_image_processor-1.log",
"/var/log/spratshop_image_processor-2.log",
"/var/log/spratshop_image_processor-3.log",
"/var/log/spratshop_image_processor-4.log",
"/var/log/spratshop_image_processor.log"
]
}
amqp {
host => ...
user => ...
password => ...
name => "from-vsftpd"
key => "spratshop"
type => "from-vsftpd"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_from-vsftpd_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "incoming-images"
key => "spratshop"
type => "incoming-images"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_incoming-images_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "processed-images"
key => "spratshop"
type => "processed-images"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_processed-images_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "send-file-todo"
key => "spratshop"
type => "send-file-todo"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_send-file-todo_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "sent-files"
key => "spratshop"
type => "sent-files"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_sent-files_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "status-checker-to-system"
key => "spratshop"
type => "status-checker-to-system"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_status-checker-to-system_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
amqp {
host => ...
user => ...
password => ...
name => "status-system-to-checker"
key => "spratshop"
type => "status-system-to-checker"
# message_format => ""
durable => true
exchange_type => "direct"
format => "json"
queue_durable => false
queue_name => "logstash_status-system-to-checker_spratshop"
# port => ... # number, default: 5672
# ssl => ... # boolean
# tags => ... # array
#
# verify_ssl => ... # boolean
# vhost => ... # string, default: "/"
}
}
filter {
grok {
type => "linux-syslog"
pattern => ["%{SYSLOG_SUDO}", "%{SYSLOG_KERNEL}", "%{SYSLOGLINE}" ]
}
date {
type => "linux-syslog"
timestamp => "MMM dd HH:mm:ss"
timestamp8601 => ISO8601
}
grok {
type => "apache-access"
pattern => "%{COMBINEDAPACHELOG}"
}
# grok {
# type => "apache-error"
# match {
# "APACHE_LOG_LEVEL" => "(?:emerg|alert|crit|error|warn|notice|info|debug)"
# "APACHE_ERROR_LOG" => "\[%{DATESTAMP_OTHER:timestamp}\] \[%{APACHE_LOG_LEVEL:level}\] %{GREEDYDATA:message}"
# }
# pattern => "%{APACHE_ERROR_LOG}"
# }
multiline {
type => "spratshop-file-sender"
pattern => "^\\s"
what => "previous"
}
multiline {
type => "spratshop-image-processor"
pattern => "^\\s"
what => "previous"
}
multiline {
type => "spratshop-status-checker"
pattern => "^\\s"
what => "previous"
}
multiline {
type => "spratshop-rails"
pattern => "^$"
negate => true
what => next
}
grep {
type => "spratshop-rails"
match => [
"@message", "^$"
]
negate => true
}
multiline {
type => "spratshop-ftp-retry-cron"
pattern => "^$"
negate => true
what => next
}
}
output {
# Emit events to stdout for easy debugging of what is going through
# logstash.
# stdout { }
# This will use elasticsearch to store your logs.
# The 'embedded' option will cause logstash to run the elasticsearch
# server in the same process, so you don't have to worry about
# how to download, configure, or run elasticsearch!
# elasticsearch {
# cluster => "default"
# host => "127.0.0.1"
# bind_host => "127.0.0.1"
#}
amqp {
host => ...
user => ...
password => ...
exchange_type => "fanout"
name => "rawlogs"
durable => true
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment