Serca ELK

logstash.conf

#####################################################################
# Fichier de configuration logstash pour les logs serca tomcat
# claude.seguret@laposte.fr, frederic.soulie@laposte.fr
######################################################################

# Outil indispensable pour groker (regexp du log à dépouper)
# https://grokdebug.herokuapp.com/discover?#

# Exemple de log
# 10/Feb/2017:00:00:05 +0100] 10.91.64.28 GET /services/mascadia/legacy/controle?idClient=EBOUTIQUE&passwdClient=pass HTTP/1.1 ?idClient=EBOUTIQUE&passwdClient=pass 200 1795 254

# Lancement
# bin/logstash -f logstash.conf
# cat mon.log |  bin/logstash -f logstash.conf

input { 
	file {
		path => ["/cygdrive/c/Users/WYH295/Downloads/logstash-5.2.0.tar/logstash-5.2.0/logstash-5.2.0/input/access.2017-02-10-head.log"]
		start_position => "beginning"
		ignore_older => 0
		sincedb_path => "/dev/null/sincedb"
	}
	stdin { }
}

filter {

  grok {
	# match => { "message" => "%{HTTPDATE:timestamp}] %{IPV4:client_ip} %{DATA:http_verb} %{DATA:request_page}(?:\?%{DATA:params}|) HTTP%{URIPATHPARAM} (?:\?%{DATA:more_params}|) %{NUMBER:http_code} (?:%{NUMBER:bytes_out}|-) (?:%{NUMBER:time_ms}|-)" }
	
	# ajout de typage pour elasticsearch, le typage permet ensuite de faire des calculs sur les types numeriques
	# float, int
	match => { "message" => "%{HTTPDATE:timestamp}] %{IPV4:client_ip} %{DATA:http_verb} %{DATA:request_page}(?:\?%{DATA:params}|) HTTP%{URIPATHPARAM} (?:\?%{DATA:more_params}|) %{NUMBER:http_code:int} (?:%{NUMBER:bytes_out:int}|-) (?:%{NUMBER:time_ms:int}|-)" }
  }
  
  # transforme les caracteres encodes en caracteres lisibles : accents, separateurs....
  urldecode {
	field => "params"
  }
  
  # Decoupe les parametres d url en plusieurs champs
  kv { 
	field_split => "&" 
	source => "params" 
  } 
  
  # Convertit le champ nbItems en numerique
  mutate {
	convert => { "nbItems" => "integer" }
  }
  
  mutate {
  	
	# suppresion des champs inutiles
	remove_field => [ "passwdClient" ]
	remove_field => [ "message" ]
	remove_field => [ "more_params" ]
	remove_field => [ "params" ]

	# pour les tests
	#remove_field => [ "http_code" ]
	#remove_field => [ "client_ip" ]
	#remove_field => [ "http_verb" ]
	#remove_field => [ "bytes_out" ]
	#remove_field => [ "time_ms" ]
	#remove_field => [ "idClient" ]
	#remove_field => [ "timestamp" ]
	#remove_field => [ "host" ]
	#remove_field => [ "ide" ]
	#remove_field => [ "ligne1" ]
	#remove_field => [ "ligne2" ]
	#remove_field => [ "ligne3" ]
	#remove_field => [ "ligne4" ]
	#remove_field => [ "ligne5" ]
	#remove_field => [ "ligne6" ]
	#remove_field => [ "ligne7" ]
	#remove_field => [ "typeResultat" ]
	#remove_field => [ "nbItems" ]
	#remove_field => [ "chaineRecherche" ]
	#remove_field => [ "testCivilite" ]
	#remove_field => [ "testCedex" ]
	#remove_field => [ "testLigne3" ]
	#remove_field => [ "testReex" ]
	#remove_field => [ "civilite" ]
	#remove_field => [ "prenom" ]
	#remove_field => [ "nom" ]
	#remove_field => [ "@timestamp" ]
	#remove_field => [ "@version" ]
	
	# transformation de caracteres + en espaces
	gsub => [
	#  # replace all +  with space
		"chaineRecherche", "\+", " ",
		"ligne1", "\+", " ",
		"ligne2", "\+", " ",
		"ligne3", "\+", " ",
		"ligne4", "\+", " ",
		"ligne5", "\+", " ",
		"ligne6", "\+", " ",
		"ligne7", "\+", " ",
		"nom", "\+", " ",
		"prenom", "\+", " ",
		"civilite", "\+", " "
	]
  }
  
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}
output {

   # export vers elasticsearch dans l'index par defaut logstash
  elasticsearch { hosts => ["localhost:9200"] }
  
  # export vers sortie standart
  stdout { codec => rubydebug }
  #csv {
  #  fields => ["timestamp", "http_verb", "request_page", "http_code", "bytes_out", "time_ms", "idClient"]
  #	 path => ["/cygdrive/c/Users/WYH295/Downloads/logstash-5.2.0.tar/logstash-5.2.0/logstash-5.2.0/data/out.csv"]
  #}
  csv {
    fields => ["request_page"]
  	 path => ["/cygdrive/c/Users/WYH295/Downloads/logstash-5.2.0.tar/logstash-5.2.0/logstash-5.2.0/input/out.csv"]
  }
  
}