auth-filter.conf

filter {
  grok {
    add_tag => [ "valid" ]

    # Example log entries for both failed and successful logins:
    #
    # Aug  9 09:13:25 vmubu01 sshd[5761]: Failed password for root from 218.87.111.109 port 45712 ssh2
    # Aug  9 09:13:31 vmubu01 sshd[5761]: message repeated 2 times: [ Failed password for root from 218.87.111.109 port 45712 ssh2]
    # Aug 14 17:25:47 vmubu01 sshd[22101]: Failed password for invalid user test from 115.68.23.130 port 43092 ssh2
    # Aug 16 13:47:44 vmubu01 sshd[730]: Accepted publickey for username from 192.168.1.225 port 38783 ssh2: RSA 01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:0f:10
    # Aug 16 13:47:57 vmubu01 sshd[816]: Accepted password for username from 192.168.1.225 port 38786 ssh2
    match => [
      "message", "%{SYSLOGTIMESTAMP:syslog_date} %{SYSLOGHOST:syslog_host} %{DATA:syslog_program}(?:\[%{POSINT}\])?: %{WORD:login} password for %{USERNAME:username} from %{IP:ip} %{GREEDYDATA}",
      "message", "%{SYSLOGTIMESTAMP:syslog_date} %{SYSLOGHOST:syslog_host} %{DATA:syslog_program}(?:\[%{POSINT}\])?: message repeated 2 times: \[ %{WORD:login} password for %{USERNAME:username} from %{IP:ip} %{GREEDYDATA}",
      "message", "%{SYSLOGTIMESTAMP:syslog_date} %{SYSLOGHOST:syslog_host} %{DATA:syslog_program}(?:\[%{POSINT}\])?: %{WORD:login} password for invalid user %{USERNAME:username} from %{IP:ip} %{GREEDYDATA}",
      "message", "%{SYSLOGTIMESTAMP:syslog_date} %{SYSLOGHOST:syslog_host} %{DATA:syslog_program}(?:\[%{POSINT}\])?: %{WORD:login} %{WORD:auth_method} for %{USERNAME:username} from %{IP:ip} %{GREEDYDATA}"
    ]
  }

  if "valid" not in [tags] {
    drop { }
  }

  mutate {
    remove_tag => [ "valid" ]
    lowercase => [ "login" ]
  }
  

  date {
    match => [ "syslog_date", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
    timezone => "Europe/Helsinki"
  }

  geoip {
    source => "ip"
  }
}

ELK.sh

#!/bin/bash

sudo apt update
sudo apt-get install apt-transport-https

# Java
sudo apt-get install openjdk-8-jdk

# oracle
# sudo add-apt-repository ppa:webupd8team/java
# sudo apt update
# sudo apt install oracle-java8-installer
# sudo apt install oracle-java8-set-default

# Elasticsearch
# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/deb.html
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list

sudo apt update
sudo apt install elasticsearch
# /etc/elasticsearch/elasticsearch.yml

# autostart
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service

# sudo systemctl start elasticsearch.service
# sudo systemctl stop elasticsearch.service

# Logstash
# https://www.elastic.co/guide/en/logstash/5.6/installing-logstash.html
# Logstash requires Java 8, neither 9 nor 10
sudo apt install logstash
# sudo systemctl start logstash.service

# Kibana
# https://www.elastic.co/guide/en/kibana/5.6/deb.html
sudo apt install kibana

# /etc/kibana/kibana.yml
sudo systemctl restart kibana

# sudo /bin/systemctl daemon-reload
# sudo /bin/systemctl enable kibana.service
# sudo systemctl start kibana.service
# sudo systemctl stop kibana.service

elk_docker.sh

#!/bin/bash

sudo docker pull sebp/elk
# sudo docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk sebp/elk

mkdir elk
cd elk

cat > ./docker-compose.yml <<EOF
elk:
  image: sebp/elk
  ports:
    - "5601:5601"
    - "9200:9200"
    - "5044:5044"
EOF


sudo sysctl -w vm.max_map_count=262144
# https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
# sudo vim /etc/sysctl.conf
# sysctl -p
# sysctl vm.max_map_count
sudo docker-compose up

filebeat-docker.conf

filebeat.inputs:
- type: log
  enabled: true
  paths:
   - /var/lib/docker/containers/*/*.log  

output.logstash:
  hosts: ["localhost:5044"]
  ssl.certificate_authorities: ["/etc/pki/tls/certs/logstash-beats.crt"]

filebeat-system.conf

input {
  beats {
    port => 5044
    host => "0.0.0.0"
  }
}
filter {
  if [fileset][module] == "system" {
    if [fileset][name] == "auth" {
      grok {
        match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
                  "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
        pattern_definitions => {
          "GREEDYMULTILINE"=> "(.|\n)*"
        }
        remove_field => "message"
      }
      date {
        match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
      }
      geoip {
        source => "[system][auth][ssh][ip]"
        target => "[system][auth][ssh][geoip]"
      }
    }
    else if [fileset][name] == "syslog" {
      grok {
        match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
        pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
        remove_field => "message"
      }
      date {
        match => [ "[system][syslog][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
      }
    }
  }
}
output {
  elasticsearch {
    hosts => localhost
    manage_template => false
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
  }
}

filebeat.sh

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.3.2-amd64.deb
sudo dpkg -i filebeat-6.3.2-amd64.deb

# https://github.com/spujadas/elk-docker
# CA cert
sudo mkdir -p /etc/pki/tls/certs
sudo cp logstash-beats.crt /etc/pki/tls/certs/logstash-beats.crt

sudo vim /etc/filebeat/filebeat.yml
sudo chmod 644 /etc/filebeat/filebeat.yml

sudo service filebeat start
sudo filebeat -e

filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/syslog
    - /var/log/auth.log

output.logstash:
  hosts: ["localhost:5044"]
  ssl.certificate_authorities: ["/etc/pki/tls/certs/logstash-beats.crt"]
 

rsyslog-input.conf

input {
  tcp {
    port => "8514"
  }
}

shadowsocks-filter.conf

filter {
  if ['program'] == "ssserver" or ['program'] == 'ss-server' {
    grok {
      match => {"message" => [
        "%{SYSLOGTIMESTAMP:[timestamp]} %{SYSLOGHOST:[hostname]} sshd(?:\[%{POSINT:[pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}"

        remove_field => "message"
      }
      date {
        match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
      }
      geoip {
        source => "[system][auth][ssh][ip]"
        target => "[system][auth][ssh][geoip]"
      }
    }
    

syslog-tcp-config.md

https://computingforgeeks.com/how-to-configure-rsyslog-centralized-log-server-on-ubuntu-18-04-lts/

## @@ means tcp while @ means udp
sudo echo "*.*     @@10.130.203.4:514" >> /etc/rsyslog.conf
sudo systemctl rsyslog restart

https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/input/syslog.html

use

input {
  tcp {
    port => "8514"
  }
}

filter {
  grok {
    match => ["message", %{SYSLOGLINE} ]
  }
  syslog_pri { }
}

have better performance than

input
{
    syslog 
    {
        port =>"514"
    }
}

syslog-tcp-filter.conf

filter {
  grok {
    match => ["message", %{SYSLOGLINE} ]
  }
}