Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
CakePHP 3 + Syslog + Logstash + Elasticsearch, these gists are source for a blogpost regarding the out of the box syslog engine for CakePHP 3 and using logstash to push them to elasticsearch
# Change default log engine at the end of bootstrap.php
# See https://book.cakephp.org/3.0/en/core-libraries/logging.html#logging-to-syslog
Log::config('default', [
'engine' => 'Syslog',
'flag' => LOG_ODELAY | LOG_PERROR,
'facility' => LOG_LOCAL7, // Use local7 as dedicated syslog facilty for this app (https://en.wikipedia.org/wiki/Syslog#Facility)
'prefix' => 'MY_APP'
]);
{
"_index": "logstash-2017.01.04",
"_type": "syslog",
"_id": "AVlp9VLfjrqyulk7dql7",
"_score": 1,
"_source": {
"@timestamp": "2017-01-04T14:50:01.685Z",
"message": "<187>Jan 4 15:50:00 ubuntu MY_APP: error: [Cake\\Network\\Exception\\InternalErrorException] No user account found in header or REMOTE_USER var. Is proxy forwarding working ok?\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Request URL: /some-url\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Stack Trace:\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #10 {main}\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: \n<187>Jan 4 15:38:10 ubuntu MY_APP: error: ",
"@version": "1",
"tags": [
"multiline",
"cakephp_log"
],
"host": "192.168.0.99",
"port": 60677,
"type": "syslog",
"timestamp": "Jan 4 15:50:00",
"logsource": "ubuntu",
"program": "MY_APP",
"loglevel": "error",
"exception": "Cake\\Network\\Exception\\InternalErrorException",
"path": "/some-url",
"stacktrace": "\n#0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n#1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n#2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n#3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n#4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n#5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n#6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n#7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n#8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n#9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n#10 {main}\n\n"
},
"fields": {
"@timestamp": [
1483541401685
]
}
}
# File: /etc/logstash/conf.d/my_app.conf
input {
tcp {
codec => multiline {
# Merge lines based on an exception
pattern => "\[%{GREEDYDATA}\]"
negate => "true"
what => "previous"
}
port => 5140
type => "syslog"
}
}
filter{
grok{
match => {"message"=>"%{SYSLOGBASE2} %{LOGLEVEL:loglevel}: \[%{DATA:exception}\] %{GREEDYDATA} Request URL: %{URIPATHPARAM:path}%{GREEDYDATA}Stack Trace:%{GREEDYDATA:stacktrace}"}
overwrite =>["message"]
add_tag => ["cakephp_log"]
}
mutate {
# Remove the prefixed syslog base (e.g.: 'Jan 4 15:39:35 ubuntu my_app: error:')
gsub => ["stacktrace","...\ ..\ ..\:..\:..\ .*\ .*\:\ error: ",""]
# Remove the syslog prefix (e.g.: '<187>')
gsub => ["stacktrace","<[0-9]+>",""]
}
}
output {
# Send to Elasticsearch
elasticsearch {
hosts => "es.mydomain.com"
index => "logstash-%{+YYYY.MM.dd}"
}
# Debug
stdout { codec => rubydebug }
}
FILE: /var/log/my_app.log
Jan 4 15:50:01 ubuntu MY_APP: error: [Cake\Network\Exception\InternalErrorException] Holy cow, something went wrong!
Jan 4 15:50:01 ubuntu MY_APP: error: Request URL: /some-url
Jan 4 15:50:01 ubuntu MY_APP: error: Stack Trace:
Jan 4 15:50:01 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\LdapAuthenticate\Auth\LdapAuthenticate->getUser(Object(Cake\Network\Request))
Jan 4 15:50:01 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\Controller\Component\AuthComponent->_getUser()
Jan 4 15:50:01 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\Controller\Component\AuthComponent->authCheck(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\Controller\Component\AuthComponent->startup(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\Event\EventManager->_callListener(Array, Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\Event\EventManager->dispatch(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\Controller\Controller->dispatchEvent('Controller.star...')
Jan 4 15:50:01 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\Controller\Controller->startupProcess()
Jan 4 15:50:01 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\Routing\Dispatcher->_invoke(Object(App\Controller\AppController))
Jan 4 15:50:01 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\Routing\Dispatcher->dispatch(Object(Cake\Network\Request), Object(Cake\Network\Response))
Jan 4 15:50:01 ubuntu MY_APP: error: #10 {main}
# File: /etc/rsyslog.d/my_app.conf
# Use local7 as log facility for my_app, send the logs to both a Logstash server over TCP (@@) and a local file in /var/log
local7.* @@logstash.mydomain.com:5140
local7.* /var/log/my_app.log
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment