Last active
May 17, 2017 11:43
-
-
Save aljoantony/42e9b1920a49d8a5c5b6 to your computer and use it in GitHub Desktop.
ELK
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ########### | |
| # Script install and configure ELK stack Server and can be used for adding new clients to this stack. | |
| # Last modified 3/3/2016 | |
| # Author: Aljo Antony | |
| ########### | |
| # Define colors in case we decide to use them | |
| color() { | |
| Red='\e[0;31m' | |
| Blue='\e[0;34m' | |
| Cyan='\e[0;36m' | |
| Nc='\e[0m' # No Color | |
| Black='\e[0;30m' | |
| Dark_Gray='\e[1;30m' | |
| Light_Blue='\e[1;34m' | |
| Green='\e[0;32m' | |
| Light_Green='\e[1;32m' | |
| Light_Cyan='\e[1;36m' | |
| Light_Red='\e[1;31m' | |
| Purple='\e[0;35m' | |
| Light_Purple='\e[1;35m' | |
| Brown='\e[0;33m' | |
| Yellow='\e[1;33m' | |
| Light_Gray='\e[0;37m' | |
| White='\e[1;37m' | |
| return 0 | |
| } | |
| # ==================== Configure_Repositories function ============================== | |
| configure_repo() { | |
| printf "${Green}Configuring Repositories...\n${Nc}" | |
| # --------------------------------------------------------------------- | |
| rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch | |
| # --------------------------------------------------------------------- | |
| cat <<END >/etc/yum.repos.d/elasticsearch.repo | |
| [elasticsearch-2.x] | |
| name=Elasticsearch repository for 2.x packages | |
| baseurl=http://packages.elastic.co/elasticsearch/2.x/centos | |
| gpgcheck=1 | |
| gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch | |
| enabled=1 | |
| END | |
| cat <<END >/etc/yum.repos.d/kibana.repo | |
| [kibana-4.4] | |
| name=Kibana repository for 4.4.x packages | |
| baseurl=http://packages.elastic.co/kibana/4.4/centos | |
| gpgcheck=1 | |
| gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch | |
| enabled=1 | |
| END | |
| cat <<END >/etc/yum.repos.d/nginx.repo | |
| [nginx] | |
| name=nginx repo | |
| baseurl=http://nginx.org/packages/rhel/6/x86_64/ | |
| gpgcheck=0 | |
| enabled=1 | |
| END | |
| cat <<END >/etc/yum.repos.d/logstash.repo | |
| [logstash-2.2] | |
| name=logstash repository for 2.2 packages | |
| baseurl=http://packages.elasticsearch.org/logstash/2.2/centos | |
| gpgcheck=1 | |
| gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch | |
| enabled=1 | |
| END | |
| # --------------------------------------------------------------------- | |
| printf "${Green}Downloading prerequisites for ELK stack...\n${Nc}" | |
| cd $ELK_DOWNLOAD_FILES | |
| wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u65-b17/jdk-8u65-linux-x64.rpm" | |
| curl -L -O https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip | |
| curl -O https://gist.githubusercontent.com/thisismitch/3429023e8438cc25b86c/raw/d8c479e2a1adcea8b1fe86570e42abab0f10f364/filebeat-index-template.json | |
| } | |
| # ==================== Installing Prerequisites function ============================== | |
| install_components() { | |
| printf "${Green}Installing required components for ELK stack...\n${Nc}" | |
| sed -i s/SELINUX=enforcing/SELINUX=disabled/g /etc/selinux/config | |
| yum clean all >>$INSTALL_LOG | |
| cd $ELK_DOWNLOAD_FILES | |
| yum install jdk-8u65-linux-x64.rpm -y 2>>$INSTALL_LOG >>$INSTALL_LOG | |
| x=("elasticsearch" "kibana" "nginx" "httpd-tools" "logstash") | |
| y=("elasticsearch-2.x" "kibana-4.4" "nginx" "null" "logstash-2.2") | |
| for i in {0..4} | |
| do | |
| install_check | |
| done | |
| } | |
| install_check() { | |
| yum list installed ${x[$i]} 2>>$INSTALL_LOG >>$INSTALL_LOG | |
| if [ "$?" = "0" ]; then | |
| tput setaf 2; echo "Application ${x[$i]} is already Installed...";tput sgr 0 | |
| else | |
| tput setaf 1; echo "Installing Application ${x[$i]} ...";tput sgr 0 | |
| install_app | |
| fi | |
| } | |
| install_app() { | |
| if [ "$i" = "3" ] ; then | |
| yum install ${x[$i]} -y >>$INSTALL_LOG | |
| else | |
| yum --enablerepo="${y[$i]}" install ${x[$i]} -y >>$INSTALL_LOG | |
| fi | |
| if [ "$?" = "0" ]; then | |
| tput setaf 2; echo "Application ${x[$i]} Installed successfully...";tput sgr 0 | |
| else | |
| error_exit | |
| fi | |
| } | |
| # ==================== Configuring Components function ============================== | |
| config_components() { | |
| printf "${Green}Configuring ELK Components...\n${Nc}" | |
| config_elastic | |
| config_kibana | |
| config_nginx | |
| config_ssl | |
| config_logstash | |
| load_kibana | |
| load_filebeat | |
| config_firewall | |
| } | |
| # ==================== Configure Elastic Search function ============================== | |
| config_elastic() { | |
| sed -i_bac 's/#.*network.host.*$/network.host: localhost/' /etc/elasticsearch/elasticsearch.yml | |
| chkconfig --add elasticsearch | |
| service elasticsearch start | |
| printf "${Green}Configured ElasticSearch successfully...\n${Nc}" | |
| } | |
| # ==================== Configure Kibana function ============================== | |
| config_kibana() { | |
| sed -i_bac 's/#.*server.host.*$/server.host: "localhost"/' /opt/kibana/config/kibana.yml | |
| chkconfig --add kibana | |
| service kibana start | |
| printf "${Green}Configured Kibana successfully...\n${Nc}" | |
| } | |
| # ==================== Configure Nginx function ============================== | |
| config_nginx() { | |
| printf "${Yellow}Enter a password for Kibana Administrator User ${Green}(kibanaadmin):\n${Nc}" | |
| htpasswd -c /etc/nginx/htpasswd.users kibanaadmin | |
| cp -p /etc/nginx/nginx.conf{,.bak} | |
| cat <<END >/etc/nginx/nginx.conf | |
| user nginx; | |
| worker_processes 1; | |
| error_log /var/log/nginx/error.log warn; | |
| pid /var/run/nginx.pid; | |
| events { | |
| worker_connections 1024; | |
| } | |
| http { | |
| include /etc/nginx/mime.types; | |
| default_type application/octet-stream; | |
| log_format main ' - [] ' | |
| ' ' | |
| ' '; | |
| access_log /var/log/nginx/access.log main; | |
| sendfile on; | |
| #tcp_nopush on; | |
| keepalive_timeout 65; | |
| #gzip on; | |
| include /etc/nginx/conf.d/*.conf; | |
| } | |
| END | |
| cat <<END >/etc/nginx/conf.d/kibana.conf | |
| server { | |
| listen 80; | |
| server_name $SERVER_NAME; | |
| auth_basic "Restricted Access"; | |
| auth_basic_user_file /etc/nginx/htpasswd.users; | |
| location / { | |
| proxy_pass http://localhost:5601; | |
| proxy_http_version 1.1; | |
| proxy_set_header Upgrade \$http_upgrade; | |
| proxy_set_header Connection 'upgrade'; | |
| proxy_set_header Host \$host; | |
| proxy_cache_bypass \$http_upgrade; | |
| } | |
| } | |
| END | |
| chkconfig --add nginx | |
| service nginx start | |
| printf "${Green}Configured Nginx successfully...\n${Nc}" | |
| } | |
| # ==================== SSL Configuration function ============================== | |
| config_ssl() { | |
| cp -p /etc/pki/tls/openssl.cnf{,.bak} | |
| sed -i_bac "/^\[ v3_ca \]/a \subjectAltName = IP: $SERVER_IP" /etc/pki/tls/openssl.cnf | |
| cd /etc/pki/tls | |
| openssl req -config /etc/pki/tls/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt >>$INSTALL_LOG | |
| printf "${Green}Configured SSL successfully...\n${Nc}" | |
| } | |
| # ==================== Logstash Configuration function ============================== | |
| config_logstash() { | |
| cat <<END >/etc/logstash/conf.d/02-beats-input.conf | |
| input { | |
| beats { | |
| port => 5044 | |
| ssl => true | |
| ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt" | |
| ssl_key => "/etc/pki/tls/private/logstash-forwarder.key" | |
| } | |
| } | |
| END | |
| cat <<END >/etc/logstash/conf.d/10-syslog-filter.conf | |
| filter { | |
| if [type] == "syslog" { | |
| grok { | |
| match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } | |
| add_field => [ "received_at", "%{@timestamp}" ] | |
| add_field => [ "received_from", "%{host}" ] | |
| } | |
| syslog_pri { } | |
| date { | |
| match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] | |
| } | |
| } | |
| } | |
| END | |
| cat <<END >/etc/logstash/conf.d/30-elasticsearch-output.conf | |
| output { | |
| elasticsearch { | |
| hosts => ["localhost:9200"] | |
| sniffing => true | |
| manage_template => false | |
| index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" | |
| document_type => "%{[@metadata][type]}" | |
| } | |
| } | |
| END | |
| service logstash configtest | |
| chkconfig --add logstash | |
| service logstash start | |
| printf "${Green}Configured Logstash successfully...\n${Nc}" | |
| } | |
| # ==================== Load Kibana Dashboard function ============================== | |
| load_kibana() { | |
| printf "${Yellow}Load Kibana Dashboard...\n${Nc}" | |
| cd $ELK_DOWNLOAD_FILES | |
| unzip beats-dashboards-*.zip >>$INSTALL_LOG | |
| cd beats-dashboards-* | |
| sh ./load.sh >>$INSTALL_LOG | |
| printf "${Light_Cyan}Kibana Dashboard loaded...\n${Nc}" | |
| } | |
| # ==================== Load Filebeat function ============================== | |
| load_filebeat() { | |
| printf "${Light_Cyan}Load File beat...\n${Nc}" | |
| cd $ELK_DOWNLOAD_FILES | |
| curl -XPUT 'http://localhost:9200/_template/filebeat?pretty' -d@filebeat-index-template.json | |
| printf "${Light_Cyan}ELK Server is ready to receive Filebeat data, let's move onto setting up Filebeat on each client server.\n" | |
| printf "${Light_Green}Connect to Kibana DashBoard using link ${Light_Red}\"http://`uname -n`\"\n${Light_Green}After entering the "kibanaadmin" credentials, you should see a page prompting you to configure a default index pattern.\n${Nc}" | |
| } | |
| # ==================== Configure firewall function ============================== | |
| config_firewall() { | |
| printf "${Light_Red}Configuring firewall on ELK Server...${Nc}\n" | |
| for p in 5044 9200 5601 80 | |
| do | |
| iptables -I INPUT -j ACCEPT -p tcp --dport $p >>$INSTALL_LOG | |
| iptables -I INPUT -j ACCEPT -p udp --dport $p >>$INSTALL_LOG | |
| iptables -I OUTPUT -j ACCEPT -p tcp --dport $p >>$INSTALL_LOG | |
| iptables -I OUTPUT -j ACCEPT -p udp --dport $p >>$INSTALL_LOG | |
| done | |
| /etc/init.d/iptables save | |
| /etc/init.d/iptables restart | |
| printf "${Light_Green}Firewall is configured successfully...${Nc}" | |
| } | |
| # ==================== Configure Client function ============================== | |
| config_client() | |
| { | |
| tput setaf 2; read -rp "Enter Client Server Private IP: " CLIENT_IP ; tput sgr 0 | |
| ssh-keygen -t rsa -f /root/.ssh/id_rsa -q -P "" | |
| printf "${Green}Enter credentials of Client Server:${Nc}" | |
| ssh-copy-id $CLIENT_IP | |
| echo "Installation log of Client Server $CLIENT_IP:" >> $INSTALL_LOG | |
| scp /etc/pki/tls/certs/logstash-forwarder.crt root@$CLIENT_IP:/tmp >> $INSTALL_LOG | |
| ssh $CLIENT_IP "mkdir -p /etc/pki/tls/certs; cp /tmp/logstash-forwarder.crt /etc/pki/tls/certs/" | |
| printf "${Green}Copied SSL Certficate...\n${Yellow}Installing Filebeat Package on Client Server...${Nc}" | |
| ssh $CLIENT_IP "rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch; echo -e '[beats]\nname=Elastic Beats Repository\nbaseurl=https://packages.elastic.co/beats/yum/el/\$basearch\nenabled=1\ngpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\ngpgcheck=1\n' > /etc/yum.repos.d/elastic-beats.repo; yum -y install filebeat" >> $INSTALL_LOG | |
| if [ "$?" = "0" ]; then | |
| cat <<END >/tmp/t.sh | |
| #!/bin/bash | |
| sed -i_bac -e '/- \/var/ s/^/#/' -e '/ paths:/a \ - /var/log/messages' -e '/ paths:/a \ - /var/log/secure' -e 's/#document_type:.*$/document_type: syslog/' -e '/elasticsearch:/,/# Logstash/ s/^/#/' -e 's/#logstash:/logstash:/' -e "/logstash:/,+2 s/#hosts: \[\"localhost:/hosts: \[\":/" -e '/hosts.*5044/a \ bulk_max_size: 1024' -e '/logstash:/,+30 s/#tls:/tls:/' -e '/ tls:/,+5 s/#certificate_authorities:.*/certificate_authorities: \["\/etc\/pki\/tls\/certs\/logstash-forwarder.crt"\]/' /etc/filebeat/filebeat.yml | |
| exit | |
| END | |
| scp /tmp/t.sh $CLIENT_IP:/tmp >> $INSTALL_LOG | |
| ssh $CLIENT_IP "sh /tmp/t.sh; service filebeat start; chkconfig --add filebeat;" | |
| printf "${Light_Green}Configured client successfully...${Nc}" | |
| else | |
| error_exit | |
| fi | |
| } | |
| # ==================== Exit function ============================== | |
| function error_exit() | |
| { | |
| tput setab 1; tput bold; tput setaf 7; echo "Unknown Error occured, check installation log located @ $INSTALL_LOG for more information";tput sgr 0 | |
| exit 1 | |
| } | |
| # ========================= BEGIN ========================== | |
| # ======================================================== | |
| # ======================================================== | |
| # ========================= VARIABLES INITIALIZATION ========================== | |
| color | |
| mkdir /tmp/elk_downloads 2>/dev/null | |
| ELK_DOWNLOAD_FILES=/tmp/elk_downloads | |
| SERVER_NAME=$(uname -n) | |
| touch /tmp/elk_downloads/install_log | |
| INSTALL_LOG=/tmp/elk_downloads/install_log | |
| tput setaf 2; read -rp "Enter ELK Server Private IP: " SERVER_IP; tput sgr 0 | |
| # ========================= FUNCTIONS INVOCATIONS ========================== | |
| BG_BLUE="$(tput setab 4)" | |
| BG_BLACK="$(tput setab 0)" | |
| FG_GREEN="$(tput setaf 2)" | |
| FG_WHITE="$(tput setaf 7)" | |
| # Screen size | |
| row=$(tput lines) | |
| col=$(tput cols) | |
| # Save screen | |
| tput smcup | |
| # Display menu until selection == 0 | |
| while [[ $REPLY != 0 ]]; do | |
| echo -n ${BG_BLUE}${FG_WHITE} | |
| clear | |
| tput sc; tput cup $((row/3-1)) $((col/3)); tput setab 2; tput bold; tput setaf 7; printf "Check logs @$INSTALL_LOG to know about task progress...\n"; tput rc | |
| tput sc; tput cup $((row/3)) $((col/3)); tput setab 1; tput bold; tput setaf 7; printf "Improvisation: aljo.antony@gmail.com\n"; tput rc | |
| cat<<EOF | |
| ============================== | |
| ELK Stack Installation Menu | |
| ------------------------------ | |
| Please enter your choice: | |
| (1) Configure repo | |
| (2) Install Components | |
| (3) Configure Components | |
| (4) Configure Client | |
| (0)Quit | |
| ------------------------------ | |
| EOF | |
| read -p "Enter selection [0-4] > " selection | |
| # Clear area beneath menu | |
| tput cup 10 0 | |
| echo -n ${BG_BLACK}${FG_GREEN} | |
| tput ed | |
| tput cup 11 0 | |
| tput sc; tput cup $((row/3-1)) $((col/3)); tput setab 2; tput bold; tput setaf 7; printf "Check logs @$INSTALL_LOG to know about task progress...\n"; tput rc | |
| tput sc; tput cup $((row/3)) $((col/3)); tput setab 1; tput bold; tput setaf 7; printf "Improvisation: aljo.antony@gmail.com\n"; tput rc | |
| # Act on selection | |
| case $selection in | |
| 1) configure_repo | |
| ;; | |
| 2) install_components | |
| ;; | |
| 3) config_components | |
| ;; | |
| 4) config_client | |
| ;; | |
| 0) break | |
| ;; | |
| *) echo "Invalid entry." | |
| ;; | |
| esac | |
| printf "\n\nPress any key to continue." | |
| read -n 1 | |
| done | |
| # Restore screen | |
| tput rmcup | |
| # ========================= END ========================== | |
| # ======================================================= | |
| # ======================================================= |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment