Skip to content

Instantly share code, notes, and snippets.

@mjul
Created April 4, 2018 11:05
Show Gist options
  • Save mjul/fa222838e94d72560c5cce6b50db3346 to your computer and use it in GitHub Desktop.
Save mjul/fa222838e94d72560c5cce6b50db3346 to your computer and use it in GitHub Desktop.
Elastic Search, Logstash and Kibana via docker-compose for parsing key=value style log files
version: '2'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2
volumes:
- esdata:/usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elk
logstash:
image: docker.elastic.co/logstash/logstash-oss:6.2.2
volumes:
# - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./logstash/pipeline:/usr/share/logstash/pipeline:ro
- /var/log/GDPR/myapplication:/var/log/GDPR/myapplication:ro
ports:
- "5000:5000"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elk
depends_on:
- elasticsearch
links:
- elasticsearch
kibana:
image: docker.elastic.co/kibana/kibana-oss:6.2.2
volumes:
- ./kibana/config/:/usr/share/kibana/config:ro
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
links:
- elasticsearch
networks:
elk:
driver: bridge
volumes:
esdata:
driver: local
## Save in: logstash/pipeline/gdpr-pipeline.conf
input {
file {
path => [ "/var/log/GDPR/myapplication/myapplication_gdpr.log" ]
}
}
filter {
# Parsing three types of GDPR log entries, one at a time
# Pattern for GDPR User Activity log entries
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} object=%{QS:object} object_category=%{QS:object_category} object_id=%{QS:object_id} object_attrs=%{QS:object_attrs}" }
add_tag => [ "User Activity" ]
overwrite => [ "host" ]
}
# Pattern for GDPR Access log entries
if "_grokparsefailure" in [tags] {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} duration=%{INT:duration} response_time=%{INT:response_time}" }
remove_tag => [ "_grokparsefailure" ]
add_tag => [ "Access" ]
overwrite => [ "host" ]
}
}
# Pattern for GDPR User Management log entries
if "_grokparsefailure" in [tags] {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} src_user=%{QS:src_user} user_attrs=%{QS:user_attrs}" }
remove_tag => [ "_grokparsefailure" ]
add_tag => [ "User Management" ]
overwrite => [ "host" ]
}
}
date {
match => [ "time", "ISO8601" ]
timezone => [ "Europe/Copenhagen" ]
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => [ "elasticsearch:9200" ]
user => 'elastic'
password => 'changeme'
index => "gdpr_logs"
id => "gdpr_pipeline_id"
}
}
## Save in: kibana/config/kibana.yml
# Default Kibana configuration from kibana-docker.
server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch:9200
elasticsearch.username: elastic
elasticsearch.password: changeme
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment