iamhowardtheduck

#!/bin/sh
# ------------------------------------------------------------------------------
#
# install epel gpg key
echo "Installing EPEL gpg key."
rpm --import http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
# install epel
echo "Installing EPEL repository"
yum install epel* -y
#

iamhowardtheduck

# Begin welcome screen.
clear
echo -e "\n\n\n\n\n\n\n\n\n\n\n\n"
echo "Welcome to the $(tput setaf 1)ArcSight$(tput setaf 7) Investigate for the $(tput setaf 4)Masses$(tput setaf 7) 1.0."  
echo 
echo "Before we begin, we're going to need wget & kafka."
yum install wget -y
wget http://apache.claz.org/kafka/2.0.0/kafka_2.11-2.0.0.tgz
echo "Now let's unwrap our Kafka present!"
tar xvf kafka_2.11-2.0.0.tgz

iamhowardtheduck

echo "Welcome to the ECE Auto-Install Script PART 1."
echo "DO NOT RUN THIS AS ROOT, BUT AS A USER WITH ROOT RIGHTS"
echo "IT WILL FAIL IF YOU RUN AS ROOT, SO DO NOT DO IT!"
echo "This is designed to be run on a minimal server install of CentOS 7 AFTER 'yum update' has been run."
echo "Once down, your system will reboot; once it does start PART 2."
read -n 1 -s -r -p "Press any key to continue"
sudo yum install wget -y 
sudo /sbin/grubby --update-kernel=ALL --args='cgroup_enable=memory cgroup.memory=nokmem swapaccount=1'
echo "overlay" | sudo tee -a /etc/modules-load.d/overlay.conf
sudo grub2-set-default 0

iamhowardtheduck

sudo yum install dialog -y
cmd=(dialog --radiolist "ECE: Eclipse Installation" 22 95 16)
options=(1 "Prepare system for ECE install, then reboot." off    # any option can be set to default to "on"
         2 "Install ECE, download all available versions, & create an Elastic cluster." off
         3 "Configure the Elasticsearch repository, download, install, and configure Beats." off
         4 "Make like a tree, and leave." off)
choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty)
clear
for choice in $choices
do

iamhowardtheduck

clear
## Set variables for easy coding
# Outputs
pBeatOut="sudo tee -a /etc/packetbeat/packetbeat.yml"
mBeatOut="sudo tee -a /etc/metricbeat/metricbeat.yml"
fBeatOut="sudo tee -a /etc/filebeat/filebeat.yml"
hBeatOut="sudo tee -a /etc/heartbeat/heartbeat.yml"
# Metricbeat Modules
mBeatSys="sudo tee -a /etc/metricbeat/modules.d/system.yml"
fBeatSys="sudo tee -a /etc/filebeat/modules.d/system.yml"

iamhowardtheduck

# INSTALL DEPENDENCIES
sudo yum install epel* -y && sudo yum install java-1.8* nano vim git net-tools -y && sudo yum update -y
#
#
# INSTALL ELASTIC REPO & DOWNLOAD METRIC & FILEBEAT
sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
sudo touch /etc/yum.repos.d/elasticsearch.repo
echo "[elasticsearch-7.x]" | sudo tee -a /etc/yum.repos.d/elasticsearch.repo
echo "name=Elasticsearch repository for 7.x packages" | sudo tee -a /etc/yum.repos.d/elasticsearch.repo
echo "baseurl=https://artifacts.elastic.co/packages/7.x/yum" | sudo tee -a /etc/yum.repos.d/elasticsearch.repo

iamhowardtheduck

#!/bin/sh
# ------------------------------------------------------------------------------
service elasticsearch stop
service kibana stop
service logstash stop
yum remove elasticsearch kibana logstash filebeat packetbeat metricbeat heartbeat-elastic -y
rm -rf /etc/elasticsearch
rm -rf /etc/kibana
rm -rf /etc/logstash
rm -rf /etc/yum.repos.d/elasticsearch.repo

iamhowardtheduck

{"attributes":{"fieldFormatMap":"{\"client.bytes\":{\"id\":\"bytes\"},\"client.port\":{\"id\":\"string\"},\"coredns.query.size\":{\"id\":\"bytes\"},\"coredns.response.size\":{\"id\":\"bytes\"},\"destination.bytes\":{\"id\":\"bytes\"},\"destination.port\":{\"id\":\"string\"},\"envoyproxy.upstream_service_time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\"}},\"event.duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asMilliseconds\",\"outputPrecision\":1}},\"event.severity\":{\"id\":\"string\"},\"http.request.body.bytes\":{\"id\":\"bytes\"},\"http.request.bytes\":{\"id\":\"bytes\"},\"http.response.body.bytes\":{\"id\":\"bytes\"},\"http.response.bytes\":{\"id\":\"bytes\"},\"http.response.status_code\":{\"id\":\"string\"},\"mysql.slowlog.bytes_received\":{\"id\":\"bytes\"},\"mysql.slowlog.bytes_sent\":{\"id\":\"bytes\"},\"mysql.slowlog.innodb.io_r_bytes\":{\"id\":\"bytes\"},\"mysql.slowlog.tmp_table_sizes\":{\"id\":\"bytes\"},\"nats.log.msg.bytes\":{

iamhowardtheduck

{"attributes":{"fieldFormatMap":"{\"member.photo\":{\"id\":\"url\",\"params\":{\"type\":\"img\",\"urlTemplate\":\"\"}},\"member.other_services.facebook.identifier\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://www.facebook.com/{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\

iamhowardtheduck

input {
  stdin {
    codec => json_lines
  }
}

filter {
    mutate  {
        uppercase => [ "[group][group_country]" ]
        replace => { "location" => "%{[group][group_lat]},%{[group][group_lon]}" }