Install ELK stack with Elasticsearch version 6 and Filebeat to parse syslog on Ubuntu server

elk-server-setup.sh

#!/bin/bash


if [ -z "$1" ]
  then
    echo "Pass nginx password"
    exit 1
fi


if [ -z "$2" ]
  then
    echo "Pass domain name"
    exit 1
fi

MYDOMAIN="$2"

#Verify running as root:
check_user() {
    USER_ID=$(/usr/bin/id -u)
    return $USER_ID
}


if [ "$USER_ID" > 0 ]; then
	echo "You must be a root user" 2>&1
	exit 1
fi



#Update system packages
update_system_packages() {
    echo -e "Updating Packages \n"
    echo "-----------------------------------------"
    apt-get -y update
}


#install nginx Packages
install_nginx() {
    echo "Install Nginx"
    echo  -e  "-------------------------------- \n"
    apt-get install nginx apache2-utils -y
    htpasswd -b -c /etc/nginx/htpasswd.users kibanaadmin $1
}


config_nginx() {
	rm /etc/nginx/sites-available/default
	cd /etc/nginx/sites-available || exit
    #create file
    touch  default
    #insert config
	cat <<- EOF > default
		server {
		    listen 80;

		    auth_basic "Restricted Access";
		    auth_basic_user_file /etc/nginx/htpasswd.users;

		    location / {
		        proxy_pass http://localhost:5601;
		        proxy_http_version 1.1;
		        proxy_set_header Upgrade \$http_upgrade;
		        proxy_set_header Connection 'upgrade';
		        proxy_set_header Host \$host;
		        proxy_cache_bypass \$http_upgrade;        
		    }
		}
	EOF
	
	service nginx restart
}


#check if java installed
check_java() {
    echo -e "checking if java is installed \n"
    echo    "--------------------------------"
    JAVA=$(which java | wc -l)
    JAVA_REQ=$(java -version 2> /tmp/version && awk '/version/ { gsub(/"/, "", $NF); print ( $NF < 1.8 ) ? "YES" : "NO" }' /tmp/version)
    if [ $JAVA -eq 0 ] ; then
    	apt-get install openjdk-8-jre-headless -y
		apt-get install openjdk-8-jdk-headless -y
    elif [ "$JAVA_REQ" = 'YES' ]; then
       apt-get update && apt-get install -y openjdk-8-jdk
    fi
}


#Install and Configure Elasticsearch
install_elasticsearch() {
    echo -e "Installing elasticsearch version 6.x\n"
    echo    "---------------------------"
    #import PGP key
    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
    echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list
    apt-get update && apt-get install -y elasticsearch
    echo -e "Updating start daemon \n"
    echo    "---------------------------"
    CMD=$(command -v systemctl)
    if [ $CMD > /dev/null ] ; then
    	systemctl daemon-reload
        systemctl enable elasticsearch.service
    else
        update-rc.d elasticsearch defaults 95 10
    fi
}


configure_elasticsearch() {
    echo -e "Configuring elasticsearch \n"
    echo    "---------------------------"
    cd /etc/elasticsearch/ || exit
    #bootstrap.memory_lock: true
    sed -i '/bootstrap.memory_lock:/s/^#//g' elasticsearch.yml
    #network.host: localhost
    sed -i '/network.host/anetwork.host: localhost'  elasticsearch.yml
    #http.port: 9200
    sed -i '/http.port:/s/^#//g' elasticsearch.yml
    #LimitMEMLOCK=infinity
    sed -i '/LimitMEMLOCK=/s/^#//g' /usr/lib/systemd/system/elasticsearch.service
    #MAX_LOCKED_MEMORY=unlimited
    sed -i '/MAX_LOCKED_MEMORY=/s/^#//g' /etc/default/elasticsearch
    #start service
    systemctl daemon-reload
    echo -e "Starting Service elasticsearch \n"
    echo    "------------------------------"
    service elasticsearch start  
    sleep 60
    #check if service is running
    SVC='elasticsearch'
    if ps ax | grep -v grep | grep $SVC > /dev/null ; then
        echo "Elasticsearch service is running"
    else
        echo "Elasticsearch Server is stopped - please check your installation"
        exit 1
    fi
}


#Install kibana
install_kibana() {
    echo -e "Installing kibana \n"
    echo    "---------------------------"
    #get eth IP
    #install package
    apt-get install -y kibana
    cd /etc/kibana || exit
    #server.port: 5601
    sed -i "/server.port:/s/^#//g" /etc/kibana/kibana.yml
    #The default is 'localhost', which usually means remote machines will not be able to connect.
    #server.host: "localhost"
    sed -i "/server.host/aserver.host: localhost"  /etc/kibana/kibana.yml
    #Elastic url
    sed -i '/elasticsearch.url:/s/^#//g' /etc/kibana/kibana.yml
    #start kibana
    echo -e "Updating start daemon Kibana \n"
    echo    "---------------------------"
    CMD=$(command -v systemctl)
    if [ $CMD > /dev/null ] ; then
        systemctl daemon-reload
        systemctl enable kibana.service
        systemctl start kibana.service
    else
        update-rc.d kibana defaults 95 10
        service kibana start
    fi
}


#Install and Configure Logstash
install_configure_logstash() {
	echo "Install logstash"
    echo   -e "--------------------------- \n"
    #install package
    apt-get install -y logstash

    echo "Configuring logstash"
    echo  -e  "--------------------------- \n"
    cd /etc/logstash/conf.d/ || exit
    touch 02-beats-input.conf
    touch 10-syslog-filter.conf
    touch 30-elasticsearch-output.conf

cat <<- EOF > 02-beats-input.conf
	input {
  		beats {
    		port => 5044
    		ssl => false
  		}
	}
EOF

cat <<- EOF > 10-syslog-filter.conf
	filter {
  		if [type] == "syslog" {
    	grok {
      		match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      		add_field => [ "received_at", "%{@timestamp}" ]
      		add_field => [ "received_from", "%{host}" ]
    	}
    	syslog_pri { }
    	date {
      		match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    	}
  	}
}
EOF

cat <<- EOF > 30-elasticsearch-output.conf
	output {
  		elasticsearch {
    		hosts => ["localhost:9200"]
    		sniffing => true
    		manage_template => false
    		index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    		document_type => "%{[@metadata][type]}"
  		}
	}
EOF

	service logstash start
}


test_elasticsearch_port() {
    echo "Testing if Elasticsearch is Ruuning on port 9200"
    echo  -e  "--------------------------- \n"
	PORT=9200
	URL="http://localhost:$PORT"
	# Check that Elasticsearch is running
	curl -s $URL 2>&1 > /dev/null
	if [ $? != 0 ]; then
    	echo "Unable to contact Elasticsearch on port $PORT."
    	echo "Please ensure Elasticsearch is running and can be reached at $URL"
    	exit -1
    else
    	echo -e "Service is Running \n" 
	fi
}

load_kibana_dashboards() {
	echo "Load kibana dashboard"
    echo  -e  "--------------------------- \n"
    cd ~ || exit   
    curl -L -O https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip
    apt-get -y install unzip
    unzip beats-dashboards-*.zip
    rm beats-dashboards-*.zip
    cd beats-dashboards-* || exit
    ./load.sh
}

install_filebeat_index() {
	echo "install filebeat index"
    echo  -e  "--------------------------- \n"
    cd ~ || exit
    curl -O https://gist.githubusercontent.com/rishiloyola/227b7a3c4e879bac59eaed92e39b61a1/raw/8c7283fa83ea3717759a0c9a66013732ae61ccd6/filebeat-index-template.json
    curl -H 'Content-Type: application/json' -XPUT 'http://localhost:9200/_template/filebeat?pretty' -d@filebeat-index-template.json
}


# TODO Generate SSL certificate
# TODO add rule to only allow 5601 9200 9300 5044 ports open

check_user
update_system_packages
install_nginx
config_nginx
check_java
install_elasticsearch
configure_elasticsearch
install_kibana
load_kibana_dashboards
install_configure_logstash
test_elasticsearch_port
install_filebeat_index

wait

logstash-client-setup.sh

#!/bin/bash


if [ -z "$1" ]
  then
    echo "Pass logstash server ip address"
    exit 1
fi

#check if service is running
SVC='filebeat'
if ps ax | grep -v grep | grep $SVC > /dev/null ; then
    echo "filebeat client is already running"
    exit 1
fi

echo "Installing logstash client to ship sys logs"
echo -e "----------------------------------------- \n"

echo "deb https://packages.elastic.co/beats/apt stable main" |  sudo tee -a /etc/apt/sources.list.d/beats.list

wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -

sudo apt-get update

sudo apt-get install filebeat -y


echo "Configuring logstash client"
echo -e "----------------------------------------- \n"

cd /etc/filebeat || exit

rm filebeat.yml

cat <<- EOF > filebeat.yml
	filebeat:
	  prospectors:
	    -
	      paths:
	        - /var/log/auth.log
	        - /var/log/syslog
	      #  - /var/log/*.log

	      input_type: log
	      
	      document_type: syslog

	  registry_file: /var/lib/filebeat/registry

	output:
	  logstash:
	    hosts: ["$1:5044"]
	    bulk_max_size: 1024

	shipper:

	logging:
	  files:
	    rotateeverybytes: 10485760 # = 10MB
EOF

service filebeat start
update-rc.d filebeat defaults 95 10

Source:

• Digital Ocean blog
• Script by paulo-amaral