EFK - Logs Monitoring.md

Monitoring

Pre requirements

Install Java8

sudo apt-get update
sudo apt-get install -y python-software-properties software-properties-common apt-transport-https

sudo add-apt-repository ppa:webupd8team/java -y
sudo apt-get update

sudo apt-get install -y oracle-java8-installer

Enable syslog TCP/UDP:

# -- /etc/rsyslog.conf --
*.* @127.0.0.1:5140 # - SYSLOG_PORT

# -- --

sudo service rsyslog restart

Elasticsearch

Install:

curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.3.deb

sudo dpkg -i elasticsearch-5.6.3.deb
sudo systemctl enable elasticsearch

Configuring:

cat > /etc/elasticsearch/elasticsearch.yml << EOL
cluster.name: blc-logs
network.host: 0.0.0.0
http.port: 9020
path.data: /var/lib/elasticsearch
EOL

Securing (TRUSTED_IP - hosts with fluentd)

sudo ufw allow ssh
sudo ufw allow from TRUSTED_IP to any port 9020
sudo ufw enable

Kibana

Install:

curl -O https://artifacts.elastic.co/downloads/kibana/kibana-5.6.3-amd64.deb

sudo dpkg -i kibana-5.6.3-amd64.deb
sudo systemctl enable kibana

Configuring:

cat > /etc/kibana/kibana.yml << EOL
server.port:5601
server.host:localhost
elasticsearch.url: "http://localhost:9020"
EOL

Securing:

sudo apt-get install -y nginx apache2-utils
cat > /etc/nginx/sites-available/kibana << EOL
server {
    listen 5605;
 
    #server_name elk-stack.co;
 
    auth_basic "Restricted Access";
    auth_basic_user_file /etc/nginx/.kibana-user;
 
    location / {
        proxy_pass http://localhost:5601;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }
}
EOL

ln -s /etc/nginx/sites-available/kibana /etc/nginx/sites-enabled/

sudo htpasswd -c /etc/nginx/.kibana-user admin
#TYPE YOUR PASSWORD

sudo nginx -t
sudo systemctl enable nginx
sudo systemctl restart nginx

sudo ufw allow 5605

FluentD

Install:

wget http://packages.treasuredata.com/2/ubuntu/trusty/pool/contrib/t/td-agent/td-agent_2.0.4-0_amd64.deb
sudo dpkg -i td-agent_2.0.4-0_amd64.deb

sudo apt-get install make libcurl4-gnutls-dev --yes
sudo /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-elasticsearch
sudo /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-record-reformer

Configuring:

### -- /etc/td-agent/td-agent.conf --
<source>
  @type syslog
  port $SYSLOG_PORT
  tag syslog
</source>

<source>
  @type forward
</source>

<source>
  @type tail
  path /var/log/blc-apps/*.log
  tag apps.*
  format json
  time_key time
  time_format "%Y-%m-%dT%H:%M:%SZ"
  pos_file /tmp/fluentd-apps.pos
</source>

<filter apps.**>
  @type record_transformer
  <record>
    ident ${tag_suffix[-2]}
    message ${record["msg"]}
  </record>
</filter>

<filter syslog.**>
  @type grep
  <exclude>
    key ident
    pattern (^horizon|^api|^psim|^keychain)
  </exclude>
</filter>

<match *.**>
  @type copy
  <store>
    @type elasticsearch
    host $ELASTICSEARCH_HOST
    port $ELASTICSEARCH_PORT
    include_tag_key true
    tag_key @log_name
    logstash_format true
    logstash_prefix logs
    flush_interval 10s
  </store>
 # <store> # can be used for debug
 #   @type stdout
 #   output_type json
 # </store>
</match>