edefaria

description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent
  plugins.
processors:
- grok:
    field: message
    patterns:
    - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
      - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
      %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}( %{NUMBER:temp.duration:float})?
      "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"

edefaria

if [event][module] == "nginx" {
	if [fileset][name] == "access" {
		grok {
			match => { "message" => ["(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:[nginx][access][remote_ip_list]}|%{NOTSPACE:[source][address]}) (-|%{DATA:[user][name]}) \[%{HTTPDATE:[nginx][access][time]}\] \"%{DATA:[nginx][access][info]}\" %{NUMBER:[http][response][status_code]:int} %{NUMBER:[http][response][body][bytes]:int} (%{NUMBER:[temp][request][time]:float} )?\"(-|%{DATA:[http][request][referrer]})\" \"(-|%{DATA:[user_agent][original]})\""] }
			pattern_definitions => {
				"NGINX_HOST" => "(?:%{IP:[destination][ip]}|%{HOSTNAME:[destination][domain]})(:%{NUMBER:[destination][port]:int})?"
				"NGINX_ADDRESS_LIST" => "(?:%{IP}|%{WORD})(\"?,?\s*(?:%{IP}|%{WORD}))*"
			}
		}
		if ("_grokparsefailure" not in [tags]) {

edefaria

# ====================== Index Lifecycle Management (ILM) ======================
#
# Configure index lifecycle management (ILM). These settings create a write
# alias and add additional settings to the index template. When ILM is enabled,
# output.elasticsearch.index is ignored, and the write alias is used to set the
# index name.

# Enable ILM support. Valid values are true, false, and auto. When set to auto
# (the default), the Beat uses index lifecycle management when it connects to a
# cluster that supports ILM; otherwise, it creates daily indices.

edefaria

#!/bin/bash

SERVER="gra1.logs.ovh.com"
USER="logs-xx-00000"
PASS="xxxxxxxx"
PREFIX="logs-xx-00000"
SUFFIX="elastalert"

cd /tmp
wget https://raw.githubusercontent.com/Yelp/elastalert/master/elastalert/es_mappings/6/elastalert.json https://raw.githubusercontent.com/Yelp/elastalert/master/elastalert/es_mappings/6/elastalert_error.json https://raw.githubusercontent.com/Yelp/elastalert/master/elastalert/es_mappings/6/elastalert_status.json https://raw.githubusercontent.com/Yelp/elastalert/master/elastalert/es_mappings/6/past_elastalert.json https://raw.githubusercontent.com/Yelp/elastalert/master/elastalert/es_mappings/6/silence.json

edefaria

log-format client_ip:%ci\tclient_port_int:%cp\tdate_time:%t\tfrontend_name:%ft\tbackend_name:%b\tserver_name:%s\ttime_request_int:%Tq\ttime_queue_int:%Tw\ttime_backend_connect_int:%Tc\ttime_backend_response_int:%Tr\ttime_duration_int:%Tt\thttp_status_code_int:%ST\tbytes_read_int:%B\tcaptured_request_cookie:%CC\tcaptured_response_cookie:%CS\ttermination_state:%tsc\tactconn_int:%ac\tfeconn_int:%fc\tbeconn_int:%bc\tsrvconn_int:%sc\tretries_int:%rc\tsrv_queue_int:%sq\tbackend_queue_int:%bq\tcaptured_request_headers:%hr\tcaptured_response_headers:%hs\thttp_request:%r\tmessage:%ci:%cp\ [%t]\ %ft\ %b/%s\ %Tq/%Tw/%Tc/%Tr/%Tt\ %ST\ %B\ %CC\ \ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r