Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
CakePHP 3 + Syslog + Logstash + Elasticsearch, these gists are source for a blogpost regarding the out of the box syslog engine for CakePHP 3 and using logstash to push them to elasticsearch
# Change default log engine at the end of bootstrap.php
# See https://book.cakephp.org/3.0/en/core-libraries/logging.html#logging-to-syslog
Log::config('default', [
'engine' => 'Syslog',
'flag' => LOG_ODELAY | LOG_PERROR,
'facility' => LOG_LOCAL7, // Use local7 as dedicated syslog facilty for this app (https://en.wikipedia.org/wiki/Syslog#Facility)
'prefix' => 'MY_APP'
]);
{
"_index": "logstash-2017.01.04",
"_type": "syslog",
"_id": "AVlp9VLfjrqyulk7dql7",
"_score": 1,
"_source": {
"@timestamp": "2017-01-04T14:50:01.685Z",
"message": "<187>Jan 4 15:50:00 ubuntu MY_APP: error: [Cake\\Network\\Exception\\InternalErrorException] No user account found in header or REMOTE_USER var. Is proxy forwarding working ok?\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Request URL: /some-url\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: Stack Trace:\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: #10 {main}\n<187>Jan 4 15:50:00 ubuntu MY_APP: error: \n<187>Jan 4 15:38:10 ubuntu MY_APP: error: ",
"@version": "1",
"tags": [
"multiline",
"cakephp_log"
],
"host": "192.168.0.99",
"port": 60677,
"type": "syslog",
"timestamp": "Jan 4 15:50:00",
"logsource": "ubuntu",
"program": "MY_APP",
"loglevel": "error",
"exception": "Cake\\Network\\Exception\\InternalErrorException",
"path": "/some-url",
"stacktrace": "\n#0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\\LdapAuthenticate\\Auth\\LdapAuthenticate->getUser(Object(Cake\\Network\\Request))\n#1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\\Controller\\Component\\AuthComponent->_getUser()\n#2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\\Controller\\Component\\AuthComponent->authCheck(Object(Cake\\Event\\Event))\n#3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\\Controller\\Component\\AuthComponent->startup(Object(Cake\\Event\\Event))\n#4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\\Event\\EventManager->_callListener(Array, Object(Cake\\Event\\Event))\n#5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\\Event\\EventManager->dispatch(Object(Cake\\Event\\Event))\n#6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\\Controller\\Controller->dispatchEvent('Controller.star...')\n#7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\\Controller\\Controller->startupProcess()\n#8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\\Routing\\Dispatcher->_invoke(Object(App\\Controller\\AppController))\n#9 /var/www/webroot/index.php(37): Cake\\Routing\\Dispatcher->dispatch(Object(Cake\\Network\\Request), Object(Cake\\Network\\Response))\n#10 {main}\n\n"
},
"fields": {
"@timestamp": [
1483541401685
]
}
}
# File: /etc/logstash/conf.d/my_app.conf
input {
tcp {
codec => multiline {
# Merge lines based on an exception
pattern => "\[%{GREEDYDATA}\]"
negate => "true"
what => "previous"
}
port => 5140
type => "syslog"
}
}
filter{
grok{
match => {"message"=>"%{SYSLOGBASE2} %{LOGLEVEL:loglevel}: \[%{DATA:exception}\] %{GREEDYDATA} Request URL: %{URIPATHPARAM:path}%{GREEDYDATA}Stack Trace:%{GREEDYDATA:stacktrace}"}
overwrite =>["message"]
add_tag => ["cakephp_log"]
}
mutate {
# Remove the prefixed syslog base (e.g.: 'Jan 4 15:39:35 ubuntu my_app: error:')
gsub => ["stacktrace","...\ ..\ ..\:..\:..\ .*\ .*\:\ error: ",""]
# Remove the syslog prefix (e.g.: '<187>')
gsub => ["stacktrace","<[0-9]+>",""]
}
}
output {
# Send to Elasticsearch
elasticsearch {
hosts => "es.mydomain.com"
index => "logstash-%{+YYYY.MM.dd}"
}
# Debug
stdout { codec => rubydebug }
}
FILE: /var/log/my_app.log
Jan 4 15:50:01 ubuntu MY_APP: error: [Cake\Network\Exception\InternalErrorException] Holy cow, something went wrong!
Jan 4 15:50:01 ubuntu MY_APP: error: Request URL: /some-url
Jan 4 15:50:01 ubuntu MY_APP: error: Stack Trace:
Jan 4 15:50:01 ubuntu MY_APP: error: #0 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(697): Example\LdapAuthenticate\Auth\LdapAuthenticate->getUser(Object(Cake\Network\Request))
Jan 4 15:50:01 ubuntu MY_APP: error: #1 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(292): Cake\Controller\Component\AuthComponent->_getUser()
Jan 4 15:50:01 ubuntu MY_APP: error: #2 /var/www/vendor/cakephp/cakephp/src/Controller/Component/AuthComponent.php(258): Cake\Controller\Component\AuthComponent->authCheck(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #3 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(385): Cake\Controller\Component\AuthComponent->startup(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #4 /var/www/vendor/cakephp/cakephp/src/Event/EventManager.php(355): Cake\Event\EventManager->_callListener(Array, Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #5 /var/www/vendor/cakephp/cakephp/src/Event/EventDispatcherTrait.php(78): Cake\Event\EventManager->dispatch(Object(Cake\Event\Event))
Jan 4 15:50:01 ubuntu MY_APP: error: #6 /var/www/vendor/cakephp/cakephp/src/Controller/Controller.php(495): Cake\Controller\Controller->dispatchEvent('Controller.star...')
Jan 4 15:50:01 ubuntu MY_APP: error: #7 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(109): Cake\Controller\Controller->startupProcess()
Jan 4 15:50:01 ubuntu MY_APP: error: #8 /var/www/vendor/cakephp/cakephp/src/Routing/Dispatcher.php(87): Cake\Routing\Dispatcher->_invoke(Object(App\Controller\AppController))
Jan 4 15:50:01 ubuntu MY_APP: error: #9 /var/www/webroot/index.php(37): Cake\Routing\Dispatcher->dispatch(Object(Cake\Network\Request), Object(Cake\Network\Response))
Jan 4 15:50:01 ubuntu MY_APP: error: #10 {main}
# File: /etc/rsyslog.d/my_app.conf
# Use local7 as log facility for my_app, send the logs to both a Logstash server over TCP (@@) and a local file in /var/log
local7.* @@logstash.mydomain.com:5140
local7.* /var/log/my_app.log
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.