Two VM inside Virtualbox/VmWare/Qemu/whatever.
- Client :
- OS: Ubuntu (from ubuntu-16.04.1-desktop-amd64.iso)
- Disk: 40 GB
- RAM: 4 GB
- vCPU: 2
- Hostname: whatever
- Metron :
- OS: Centos (from CentOS-6.7-x86_64-minimal.iso)
- Disk: 200 GB
- RAM: 16 GB
- vCPU: 4
- Hostname: node1
- Network: static IPv4
Update/upgrade
# yum update
# yum install lsof vim
# cat /etc/sysconfig/network
# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
TYPE=Ethernet
UUID=79427183-17bb-4980-a512-dc29319509e1
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
HWADDR=08:00:27:65:04:C4
IPADDR=192.168.1.11
PREFIX=24
GATEWAY=192.168.1.1
DNS1=8.8.8.8
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
NAME="System eth0"
[root@node1 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 8.8.8.8
[root@node1 ~]# cat /etc/hosts
127.0.0.1 node1
127.0.0.1 localhost
# reboot
Snapshot
# yum clean all
# shutdown -h now
// Snapshot "Fresh install"
Optional : Install Virtualbox guest addition
Update/upgrade
$ sudo apt update && sudo apt upgrade && sudo apt install vim openssh-server sudo && sudo apt autoremove && sudo reboot
Install basic dependencies for ansible
$ wget https://bootstrap.pypa.io/get-pip.py && sudo -H python get-pip.py
$ sudo apt install python-dev libssl-dev
$ sudo -H pip install --upgrade ansible
OR
$ sudo -H pip uninstall -y ansible && sudo -H pip install ansible==2.0.0.2
Setup connection:
$ ssh-keygen -t rsa -b 4096
$ echo "192.168.1.11 node1" | sudo tee -a /etc/hosts >/dev/null
Check connection:
$ ping node1
$ ssh root@node1
Upload ssh key and check that password is not needed anymore
$ ssh-copy-id root@node1
$ ssh root@node1
Setup dev environment:
$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt update
$ sudo apt install oracle-java8-installer git oracle-java8-set-default maven
$ . /etc/profile.d/jdk.sh
$ env |grep JAVA
$ cd && git clone https://github.com/apache/incubator-metron.git
$ cd incubator-metron && git rev-parse HEAD
23ff817ebdd3b63be69ad9b6c493b41bb0be4880
$ rm -rf ~/.m2/
Fix build metron https://issues.apache.org/jira/browse/METRON-500
$ sed -i "s#<id></id>#<id>archive</id>#g" metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml
Build metron
$ mvn clean package -DskipTests
Setup Ansible connection:
$ cd ~/incubator-metron/metron-deployment
$ sed -i '/\[defaults\]/a remote_user = root' vagrant/full-dev-platform/ansible.cfg
$ sed -i 's/eth1/eth0/g' inventory/full-dev-platform/group_vars/all
$ cd vagrant/full-dev-platform
$ ansible -i ../../inventory/full-dev-platform/hosts all -m ping
node1 | SUCCESS => {
"changed": false,
"ping": "pong"
}
Fix some stuff :
- Support of ansible 2.1 : apache/metron#219
Install everything:
$ cd ~/incubator-metron/metron-deployment/vagrant/full-dev-platform
$ ansible-playbook -i ../../inventory/full-dev-platform/hosts ../../playbooks/metron_full_install.yml --skip-tags="solr,yaf"
...
TASK [deployment-report : debug] ***********************************************
ok: [node1] => {
"Success": [
"Apache Metron deployed successfully",
" Metron @ http://node1:5000",
" Ambari @ http://node1:8080",
" Sensor Status @ http://node1:2812",
" Topology Status @ http://node1:2812",
" Zookeeper @ node1:2181",
" Kafka @ node1:6667",
"For additional information, see https://metron.incubator.apache.org/'"
]
}
PLAY RECAP *********************************************************************
node1 : ok=332 changed=125 unreachable=0 failed=0
On node 1, stop pcap replay, bro and yaf
# monit summary
# monit stop pcap-service pcap-replay pcap-parser bro bro-parser yaf-parser
# pkill -f "/opt/bin/tcpreplay --intf1=eth0 --loop=0 /opt/pcap-replay/example.pcap"
Play with snort logs
# ps faux |grep -B 1 "alert.csv"
flume 25260 0.6 0.7 3580532 114676 ? Sl 17:14 0:11 /usr/jdk64/jdk1.8.0_77/bin/java -Xmx20m ...org.apache.flume.node.Application --conf-file /usr/hdp/2.4.3.0-227/flume/conf/flume-snort.conf --name snort
flume 26567 0.0 0.0 100952 660 ? S 17:14 0:00 \_ tail -F /var/log/snort/alert.csv
# curl http://node1:9200/_cat/indices?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
yellow open .kibana 5 1 34 0 73.4kb 73.4kb
No snort index .. Possible issue: Mysql did not start.
# cat /var/log/storm/enrichment-4-1476717306-worker-6703.log
...
2016-10-17 21:17:54.247 o.a.m.e.a.j.JdbcAdapter [ERROR] [Metron] JDBC connection failed....
...
# service mysql status
# service mysql start
Restart the topo
# storm kill enrichment
# /usr/metron/0.2.1BETA/bin/start_enrichment_topology.sh
OR
# monit restart enrichment
Oh, a new index :)
$ curl http://node1:9200/_cat/indices?v