ArcSight Configurator for Elastic

ACE.sh

#!/bin/sh
# ------------------------------------------------------------------------------
# Begin welcome screen.
clear
echo -e "\n\n\n\n\n\n\n\n\n\n\n\n"
echo "Welcome to the $(tput setaf 1)ArcSight$(tput setaf 7) configurator for $(tput setaf 4)Elastic$(tput setaf 7) 2.0."  
echo 
echo "Before we begin, we're going to need a few packages."
echo 
echo "First we'll install the EPEL repository, then install the following packages:"
echo 
echo "chrony* device-mapper-libs* dialog* git htop java-1.8* keyutils* libevent* libgcrypt* libseccomp* "
echo "libtool-ltdl* lsof* nano net-tools* nfs-utils* nmap*  systemd-libs* tcpdump* unzip wget wireshark*"
echo -e "\n\n"
echo "We'll then install the following group packages:"
echo
echo "Development Tools, Web Server, & Compatibility Libraries"
echo -e "\n\n\n" 
echo "This will stop and disable your firewall, as this is in a lab...$(tput setaf 1)RIGHT?$(tput setaf 7)"
echo 
echo "So please, do not implement this in production."
echo 
echo "We'll then configure the Elastic repos and install the stack."
echo 
echo "You will be given a chance to continue configuring integration with ArcSight or exit."
echo 
echo 
echo "Enjoy! ☺"
echo -e "\n\n\n\n\n\n\n\n\n\n\n\n"
read -n 1 -s -r -p "Press any key to continue"
#
# Begin core installation.
# install epel gpg key
clear
echo "Installing EPEL gpg key." & rpm --import http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 
echo "EPEL gpg key Installed:                                          $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
echo 
# install epel
echo "Installing EPEL repository"
echo 
yum install epel* -y
echo "EPEL repository Installed:                                       $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
# install updates and needed programs
echo "Installing packages and helpful tools then updating"
echo 
yum install chrony* dialog* gamin* spawn-fcgi* device-mapper-libs* git htop java-1.8* keyutils* libevent* libgcrypt* libseccomp* libtool-ltdl*	lsof* nano net-tools* nfs-utils* systemd-libs* unzip wget -y && yum update -y
echo "Packages & Update Installed:                                     $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
# install Group packages
echo "Group installing Development Tools, Web Server, & Compatibility Libraries"
echo 
yum groupinstall "Development Tools" "Web Server" "Compatibility Libraries" -y
echo "Group Packages Installed:                                         $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
# turn off firewalld
echo "Turning off the firewall, since this is a lab; DO NOT DO THIS IN PRODUCTION! ☺"
echo 
service firewalld stop
systemctl stop firewalld
systemctl disable firewalld
echo "Firewall stopped and disabled on boot.                            $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
echo
echo
echo
read -n 1 -s -r -p "Press any key to continue"
#
# import Elastic gpg key
clear
echo "Import Elastic gpg key"
echo 
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
echo "Import Elastic gpg key:                                           $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
#
# create Elastic repo
echo "Creating Elastic, Kibana, & Logstash repositories"
echo 
touch /etc/yum.repos.d/elasticsearch.repo
echo "[elasticsearch-6.x]" >>  /etc/yum.repos.d/elasticsearch.repo
echo "name=Elasticsearch repository for 6.x packages" >> /etc/yum.repos.d/elasticsearch.repo
echo "baseurl=https://artifacts.elastic.co/packages/6.x/yum"  >> /etc/yum.repos.d/elasticsearch.repo
echo "gpgcheck=1"  >> /etc/yum.repos.d/elasticsearch.repo
echo "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch"  >> /etc/yum.repos.d/elasticsearch.repo
echo "enabled=1"  >> /etc/yum.repos.d/elasticsearch.repo
echo "autorefresh=1"  >> /etc/yum.repos.d/elasticsearch.repo
echo "type=rpm-md" >> /etc/yum.repos.d/elasticsearch.repo
#
echo "Elasticsearch REPO created:                                       $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
# create Kibana repo
touch /etc/yum.repos.d/kibana.repo
echo "[kibana-6.x]" >>  /etc/yum.repos.d/kibana.repo
echo "name=Elasticsearch repository for 6.x packages" >> /etc/yum.repos.d/kibana.repo
echo "baseurl=https://artifacts.elastic.co/packages/6.x/yum"  >> /etc/yum.repos.d/kibana.repo
echo "gpgcheck=1"  >> /etc/yum.repos.d/kibana.repo
echo "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch"  >> /etc/yum.repos.d/kibana.repo
echo "enabled=1"  >> /etc/yum.repos.d/kibana.repo
echo "autorefresh=1"  >> /etc/yum.repos.d/kibana.repo
echo "type=rpm-md" >> /etc/yum.repos.d/kibana.repo
echo "Kibana REPO created:                                              $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
# create Logstash repo
touch /etc/yum.repos.d/logstash.repo
echo "[logstash-6.x]" >>  /etc/yum.repos.d/logstash.repo
echo "name=Elasticsearch repository for 6.x packages" >> /etc/yum.repos.d/logstash.repo
echo "baseurl=https://artifacts.elastic.co/packages/6.x/yum"  >> /etc/yum.repos.d/logstash.repo
echo "gpgcheck=1"  >> /etc/yum.repos.d/logstash.repo
echo "gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch"  >> /etc/yum.repos.d/logstash.repo
echo "enabled=1"  >> /etc/yum.repos.d/logstash.repo
echo "autorefresh=1"  >> /etc/yum.repos.d/logstash.repo
echo "type=rpm-md" >> /etc/yum.repos.d/logstash.repo
echo "Logstash REPO created:                                            $(tput setaf 2)[$(tput setaf 4)OK$(tput setaf 2)]$(tput setaf 7)"
#
# install Elastic, Kibana, Logstash, Beats
yum install elasticsearch kibana logstash metricbeat filebeat packetbeat auditbeat heartbeat-elastic -y
#
#
echo "Congratulations, Elasticsearch, Kibana, Logstash, and the myriad of Beats have been installed."
#
# Begin modification of Elastic, Kibana, & Logstash yaml files, modification of JVM memory, and create a Logstash conf file.
# This script will make a copy of the original as .bak in the same folder.
# It will then create a new file with your values.
#
# 5-OCT-2018
# Added Variables to set Elastic IP, Hostname, Cluster name, Node name, Kibana IP, and Logstash configs.
#
echo "Would you like to configure Elasticsearch, Kibana, Logstash, and integrate with an ArcSight source now?"
read -p "Continue (y/n)?" choice
case "$choice" in 
  y|Y ) echo "We will now begin a simple configuration setting to configure Elastic to consume from an ArcSight source";;
  n|N ) echo "The install will now stop and rub your feet, well maybe not rub your feet.  ☺" ; ;;
  * ) echo "invalid";;
esac
#
# Begin Elastic Stack configuration.
mv /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml.bak
echo "What is the IPv4 address of this system?"
echo 
read elkIP
echo 
echo "Please set the Elastic Cluster name.  This is cosmetic only."
echo 
read elkClusterName
echo 
echo "Please set the Elastic Node name.  This is cosmetic only."
echo 
read elkNodeName
echo 
touch /etc/elasticsearch/elasticsearch.yml
echo "# ======================== Elasticsearch Configuration =========================" >> /etc/elasticsearch/elasticsearch.yml
echo -e "cluster.name: ${elkClusterName}" >> /etc/elasticsearch/elasticsearch.yml
echo -e "node.name: ${elkNodeName}" >> /etc/elasticsearch/elasticsearch.yml
echo "path.data: /var/lib/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
echo "path.logs: /var/log/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
echo -e "network.host: ${elkIP}" >> /etc/elasticsearch/elasticsearch.yml
echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml
echo "#" >> /etc/elasticsearch/elasticsearch.yml
echo "# ======================== Elasticsearch Configuration =========================" >> /etc/elasticsearch/elasticsearch.yml
#
#
# Begin JVM memory configuration.
cp /etc/elasticsearch/jvm.options /etc/elasticsearch/jvm.options.bak
echo "How much RAM would you like to allocate for Elastic?  Typically this is 50% of your physical memory."
echo 
echo "EXAMPLE INPUT: 256M OR 2G"
echo 
read elkRAM
sed -i "22s/.*/-Xms$elkRAM/" /etc/elasticsearch/jvm.options
sed -i "23s/.*/-Xmx$elkRAM/" /etc/elasticsearch/jvm.options
#
# Begin Elasticsearch services and enable at boot.
service elasticsearch start
systemctl enable elasticsearch
#
cp /etc/logstash/jvm.options /etc/logstash/jvm.options.bak
echo 
echo "How much RAM would you like to allocate for Logstash?"
echo
echo "Typically, this 5~10% of physical RAM"
echo 
echo "EXAMPLE INPUT: 256M OR 2G"
echo 
read logRAM
sed -i "6s/.*/-Xms$logRAM/" /etc/logstash/jvm.options
sed -i "7s/.*/-Xmx$logRAM/" /etc/logstash/jvm.options
echo 
#
# Begin Kibana modification.
mv /etc/kibana/kibana.yml /etc/kibana/kibana.yml.bak
echo "Please set the Kibana Server name.  This is cosmetic only."
echo 
read kibName
touch /etc/kibana/kibana.yml
echo "server.port: 5601" >> /etc/kibana/kibana.yml
echo "server.host: \"${elkIP}\"" >> /etc/kibana/kibana.yml
echo "server.name: \"${kibName}\"" >> /etc/kibana/kibana.yml
echo "elasticsearch.url: \"http://${elkIP}:9200\"" >> /etc/kibana/kibana.yml
#
# Begin Kibana services and enable at boot.
service kibana start
systemctl kibana elasticsearch
#
#
# Begin modification of Elastic & Kibana yaml files, and create Logstash conf file.
# This script will make a copy of the original as .bak in the same folder.
# It will then create a new file with your values.
# Begin ArcSight consumption configuration.
mv /etc/logstash/logstash.yml /etc/logstash/logstash.yml.bak
cmd=(dialog --separate-output --checklist "How would you like to configure Elastic's consumption of ArcSight events?" 22 76 16)
options=(1 "SmartConnectors sending CEF Syslog via a TCP Port." off    # any option can be set to default to "on"
         2 "Kafka or Event Broker topic of any non-ESM topic." off
         3 "An ArcSight sandwich of SmartConnectors & Kafka." off
         4 "Decide to be a coward, back off and exit." off)
choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
clear
for choice in $choices
do
    case $choice in
        1)	clear
		    echo
			echo
		    echo "What TCP port will your SmartConnector send CEF Syslog on?  This will be the Port parameter on your CEF Syslog destination."
			read scTCPPort
			echo "Go SmartConnectors, it's your birthday ♪♫♪"
			touch /etc/logstash/logstash.yml
			echo "xpack.monitoring.enabled: true" >> /etc/logstash/logstash.yml
			echo "xpack.monitoring.elasticsearch.url: ${elkIP}:9200" >> /etc/logstash/logstash.yml
			echo "path.data: /var/lib/logstash" >> /etc/logstash/logstash.yml
			echo "path.logs: /var/log/logstash" >> /etc/logstash/logstash.yml
			echo "path.config: /etc/logstash/conf.d/logstash.conf" >> /etc/logstash/logstash.yml
			touch /etc/logstash/logstashKickStart.sh
			echo "bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M \"arcsight.var.inputs=smartconnector\" -M \"arcsight.var.input.smartconnector.port=${scTCPPort}\" -M \"arcsight.var.elasticsearch.hosts=${elkIP}:9200\" -M \"var.elasticsearch.ssl.enabled=false\" -M \"arcsight.var.kibana.host=${elkIP}:5601\" -M \"arcsight.var.kibana.ssl.enabled=false\" -M \"arcsight.var.kibana.ssl.verfication_mode=false\" &" >> /etc/logstash/logstashKickStart.sh
			chmod +x /etc/logstash/logstashKickStart.sh
			bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M "arcsight.var.inputs=smartconnector" -M "arcsight.var.input.smartconnector.port=${scTCPPort}" -M "arcsight.var.elasticsearch.hosts=${elkIP}:9200" -M "var.elasticsearch.ssl.enabled=false" -M "arcsight.var.kibana.host=${elkIP}:5601" -M "arcsight.var.kibana.ssl.enabled=false" -M "arcsight.var.kibana.ssl.verfication_mode=false" &
			touch /etc/logstash/conf.d/logstash.conf
			echo "input {" >> /etc/logstash/conf.d/logstash.conf
			echo "       tcp {" >> /etc/logstash/conf.d/logstash.conf
			echo "            port => ${scTCPPort}" >> /etc/logstash/conf.d/logstash.conf
                        echo "            type => syslog" >> /etc/logstash/conf.d/logstash.conf
			echo "           }" >> /etc/logstash/conf.d/logstash.conf
			echo "      }" >> /etc/logstash/conf.d/logstash.conf"
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "#       Uncomment me and....       #" >> /etc/logstash/conf.d/logstash.conf
			echo "# FILTER OR GROK HERE IF NECESSARY #" >> /etc/logstash/conf.d/logstash.conf
			echo "#                                  #" >> /etc/logstash/conf.d/logstash.conf
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "output {" >> /etc/logstash/conf.d/logstash.conf
			echo "       elasticsearch { hosts => [\"${elkIP}:9200\"] }" >> /etc/logstash/conf.d/logstash.conf
			echo "      }" >> /etc/logstash/conf.d/logstash.conf
			clear
			echo
			echo "Logstash is now enabled at boot, and is initiating now."
			echo
			echo "Please open a browser and go to http://${elkIP}:5601"
			echo
			echo "The script logstashKickStart has been created here: /etc/logstash/logstashKickStart.sh"
			echo
			echo "Use this in case you want to see the command to, you guessed it, kick start Logstash."
			echo
			echo "Output of Logstash will soon commence..."
			echo 
			;;
        2)  clear
		    echo
			echo
		    echo "What CEF formatted topic will you consume from?"
			read kafkaTopic
			echo 
			echo "What is the IPv4 address of your Kafka node?"
			read kafkaIP
			echo 
			echo "Kafka can't drive 55 ♪♫♪"
                        touch /etc/logstash/logstash.yml
			echo "path.data: /var/lib/logstash" >> /etc/logstash/logstash.yml
			echo "path.logs: /var/log/logstash" >> /etc/logstash/logstash.yml
			echo "path.config: /etc/logstash/conf.d/logstash.conf" >> /etc/logstash/logstash.yml
			echo "xpack.monitoring.enabled: true" >> /etc/logstash/logstash.yml
			echo "xpack.monitoring.elasticsearch.url: \$elkIP:9200" >> /etc/logstash/logstash.yml
			touch /etc/logstash/logstashKickStart.sh
			echo "bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M \"arcsight.var.inputs=eventbroker\" -M \"arcsight.var.input.eventbroker.bootstrap_servers=${kafkaIP}:9092\" -M \"arcsight.var.input.eventbroker.topics=${kafkaTopic}\" -M \"arcsight.var.elasticsearch.hosts=${elkIP}:9200\" -M \"var.elasticsearch.ssl.enabled=false\" -M \"arcsight.var.kibana.host=${elkIP}:5601\" -M \"arcsight.var.kibana.ssl.enabled=false\" -M \"arcsight.var.kibana.ssl.verfication_mode=false\" &" >> /etc/logstash/logstashKickStart.sh
			chmod +x /etc/logstash/logstashKickStart.sh
			bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M "arcsight.var.inputs=eventbroker" -M "arcsight.var.input.eventbroker.bootstrap_servers=${kafkaIP}:9092" -M "arcsight.var.input.eventbroker.topics=${kafkaTopic}" -M "arcsight.var.elasticsearch.hosts=${elkIP}:9200" -M "var.elasticsearch.ssl.enabled=false" -M "arcsight.var.kibana.host=${elkIP}:5601" -M "arcsight.var.kibana.ssl.enabled=false" -M "arcsight.var.kibana.ssl.verfication_mode=false" &
            		touch /etc/logstash/conf.d/logstash.conf
			echo "input {" >> /etc/logstash/conf.d/logstash.conf
			echo "       kafka {" >> /etc/logstash/conf.d/logstash.conf
			echo "              bootstrap_servers => \"${kafkaIP}:9092\"" >> /etc/logstash/conf.d/logstash.conf
			echo "              topics => \"${kafkaTopic}\"" >> /etc/logstash/conf.d/logstash.conf
			echo "      }" >> /etc/logstash/conf.d/logstash.conf
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "#       Uncomment me and....       #" >> /etc/logstash/conf.d/logstash.conf
			echo "# FILTER OR GROK HERE IF NECESSARY #" >> /etc/logstash/conf.d/logstash.conf
			echo "#                                  #" >> /etc/logstash/conf.d/logstash.conf
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "output {" >> /etc/logstash/conf.d/logstash.conf
			echo "       elasticsearch { hosts => [\"${elkIP}:9200\"] }" >> /etc/logstash/conf.d/logstash.conf
			echo "      }" >> /etc/logstash/conf.d/logstash.conf
			clear
			echo
			echo "Logstash is now enabled at boot, and is initiating now."
			echo
			echo "Please open a browser and go to http://${elkIP}:5601"
			echo
			echo "The script logstashKickStart has been created here: /etc/logstash/logstashKickStart.sh"
			echo
			echo "Use this in case you want to see the command to, you guessed it, kick start Logstash."
			echo
			echo "Output of Logstash will soon commence..."
			echo 
			;;
		3)  clear
		    echo
			echo
		    echo "What TCP port will your SmartConnector send CEF Syslog on?"
			read scTCPPort
			echo 
			echo "What CEF formatted topic will you consume from?"
			read kafkaTopic
			echo 
			echo "What is the IPv4 Broker address of your Kafka node?"
			read kafkaIP
			echo 
			echo "Pour some sugar on me ♪♫♪"
                        echo "path.data: /var/lib/logstash" >> /etc/logstash/logstash.yml 
		 	echo "path.logs: /var/log/logstash" >> /etc/logstash/logstash.yml
			echo "path.config: /etc/logstash/conf.d/logstash.conf" >> /etc/logstash/logstash.yml
			echo "xpack.monitoring.enabled: true" >> /etc/logstash/logstash.yml
			echo "xpack.monitoring.elasticsearch.url: ${elkIP}:9200" >> /etc/logstash/logstash.yml
			touch /etc/logstash/logstashKickStart.sh
			echo "bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M \"arcsight.var.inputs=eventbroker,smartconnector\" -M \"arcsight.var.input.eventbroker.bootstrap_servers=${kafkaIP}:9092\" -M \"arcsight.var.input.eventbroker.topics=${kafkaTopic}\" -M \"arcsight.var.input.smartconnector.port=${scTCPPort}\" -M \"arcsight.var.elasticsearch.hosts=${elkIP}:9200\" -M \"var.elasticsearch.ssl.enabled=false\" -M \"arcsight.var.kibana.host=${elkIP}:5601\" -M \"arcsight.var.kibana.ssl.enabled=false\" -M \"arcsight.var.kibana.ssl.verficati/on_mode=false\" &" >> /etc/logstash/logstashKickStart.sh
             		chmod +x /etc/logstash/logstashKickStart.sh
			bash /usr/share/logstash/bin/logstash --modules arcsight --setup -M "arcsight.var.inputs=eventbroker,smartconnector" -M "arcsight.var.input.eventbroker.bootstrap_servers=${kafkaIP}:9092" -M "arcsight.var.input.eventbroker.topics=${kafkaTopic}" -M "arcsight.var.input.smartconnector.port=${scTCPPort}" -M "arcsight.var.elasticsearch.hosts=${elkIP}:9200" -M "var.elasticsearch.ssl.enabled=false" -M "arcsight.var.kibana.host=${elkIP}:5601" -M "arcsight.var.kibana.ssl.enabled=false" -M "arcsight.var.kibana.ssl.verfication_mode=false" &
			touch /etc/logstash/conf.d/logstash.conf
			echo "input {" >> /etc/logstash/conf.d/logstash.conf
			echo "       tcp {" >> /etc/logstash/conf.d/logstash.conf
			echo "            port => ${scTCPPort}" >> /etc/logstash/conf.d/logstash.conf
            echo "            type => syslog" >> /etc/logstash/conf.d/logstash.conf
			echo "           }" >> /etc/logstash/conf.d/logstash.conf
			echo "       kafka {" >> /etc/logstash/conf.d/logstash.conf
			echo "              bootstrap_servers => \"${kafkaIP}:9092\"" >> /etc/logstash/conf.d/logstash.conf
			echo "              topics => \"${kafkaTopic}\"" >> /etc/logstash/conf.d/logstash.conf
			echo "           }" >> /etc/logstash/conf.d/logstash.conf
			echo "       }" >> /etc/logstash/conf.d/logstash.conf
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "#       Uncomment me and....       #" >> /etc/logstash/conf.d/logstash.conf
			echo "# FILTER OR GROK HERE IF NECESSARY #" >> /etc/logstash/conf.d/logstash.conf
			echo "#                                  #" >> /etc/logstash/conf.d/logstash.conf
			echo "####################################" >> /etc/logstash/conf.d/logstash.conf
			echo "output {" >> /etc/logstash/conf.d/logstash.conf
			echo "       elasticsearch { hosts => [\"${elkIP}:9200\"] }" >> /etc/logstash/conf.d/logstash.conf
			echo "      }" >> /etc/logstash/conf.d/logstash.conf
			clear
			echo
			echo "Logstash is now enabled at boot, and is initiating now."
			echo
			echo "Please open a browser and go to http://${elkIP}:5601"
			echo
			echo "The script logstashKickStart has been created here: /etc/logstash/logstashKickStart.sh"
			echo
			echo "Use this in case you want to see the command to, you guessed it, kick start Logstash."
			echo
			echo "Output of Logstash will soon commence..."
			echo 
			;;
        4)
            echo "Later-gator!  Please reference the Elasticsearch & ArcSight Deployment Guide to complete your setup. ♪♫♪"
            exit
			;;
    esac
done

Version 1.5:

Updated Logstash configuration for proper boot as service.
Added filter or grok section for Logstash configuration.

Version 1.9:

Added logstashKickstart.sh as a bash script which is the command to start Logstash with the ArcSight values passed during install; just in case the Logstash service does not work.

Updated to 2.0, apologies for the case of the dumbs.