Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Metron full dev platform without Vagrant (Windows - 02/12/2016) -- in progress

Metron full dev platform 0.3 (Windows - 02/12/2016)

Two VM inside Virtualbox/VmWare/Qemu/whatever.

  • Client :
    • OS: Ubuntu (from ubuntu-16.04.1-desktop-amd64.iso)
    • Disk: 40 GB
    • RAM: 4 GB
    • vCPU: 2
    • Hostname: whatever
  • Metron :
    • OS: Centos (from CentOS-6.7-x86_64-minimal.iso)
    • Disk: 200 GB
    • RAM: 16 GB
    • vCPU: 4
    • Hostname: node1
    • Network: static IPv4

Metron

Update/upgrade


# yum update
# yum install lsof vim 

# cat /etc/sysconfig/network

# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
TYPE=Ethernet
UUID=79427183-17bb-4980-a512-dc29319509e1
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
HWADDR=08:00:27:65:04:C4
IPADDR=192.168.1.11
PREFIX=24
GATEWAY=192.168.1.1
DNS1=8.8.8.8
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
NAME="System eth0"

[root@node1 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 8.8.8.8

[root@node1 ~]# cat /etc/hosts
127.0.0.1 node1
127.0.0.1 localhost

# reboot

Snapshot

# yum clean all
# shutdown -h now
// Snapshot "Fresh install"

Client

Optional : Install Virtualbox guest addition

Update/upgrade

$ sudo apt update && sudo apt upgrade && sudo apt install vim openssh-server sudo && sudo apt autoremove && sudo reboot

Install basic dependencies for ansible

$ wget  https://bootstrap.pypa.io/get-pip.py && sudo -H  python get-pip.py
$ sudo apt install python-dev libssl-dev
$ sudo -H pip install --upgrade ansible
OR
$ sudo -H pip uninstall -y ansible && sudo -H pip install ansible==2.0.0.2

Setup connection:

$ ssh-keygen -t rsa -b 4096
$ echo "192.168.1.11 node1" | sudo tee -a /etc/hosts >/dev/null

Check connection:

$ ping node1
$ ssh root@node1

Upload ssh key and check that password is not needed anymore

$ ssh-copy-id root@node1
$ ssh root@node1

Setup dev environment:

$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt update
$ sudo apt install oracle-java8-installer git oracle-java8-set-default maven
$ . /etc/profile.d/jdk.sh 
$ env |grep JAVA
$ cd && git clone https://github.com/apache/incubator-metron.git
$ cd incubator-metron && git rev-parse HEAD
23ff817ebdd3b63be69ad9b6c493b41bb0be4880
$ rm -rf  ~/.m2/

Fix build metron https://issues.apache.org/jira/browse/METRON-500

$ sed -i "s#<id></id>#<id>archive</id>#g" metron-deployment/packaging/ambari/metron-mpack/src/main/assemblies/metron-mpack.xml

Build metron


$ mvn clean package  -DskipTests

Setup Ansible connection:

$ cd ~/incubator-metron/metron-deployment
$ sed -i '/\[defaults\]/a remote_user = root'  vagrant/full-dev-platform/ansible.cfg
$ sed -i 's/eth1/eth0/g' inventory/full-dev-platform/group_vars/all
$ cd vagrant/full-dev-platform
$ ansible -i ../../inventory/full-dev-platform/hosts all -m ping
node1 | SUCCESS => {
    "changed": false,
    "ping": "pong"
}

Fix some stuff :

Install everything:

$ cd ~/incubator-metron/metron-deployment/vagrant/full-dev-platform
$ ansible-playbook  -i ../../inventory/full-dev-platform/hosts ../../playbooks/metron_full_install.yml --skip-tags="solr,yaf" 
...
TASK [deployment-report : debug] ***********************************************
ok: [node1] => {
    "Success": [
        "Apache Metron deployed successfully",
        "   Metron          @ http://node1:5000",
        "   Ambari          @ http://node1:8080",
        "   Sensor Status   @ http://node1:2812",
        "   Topology Status @ http://node1:2812",
        "   Zookeeper       @ node1:2181",
        "   Kafka           @ node1:6667",
        "For additional information, see https://metron.incubator.apache.org/'"
    ]
}

PLAY RECAP *********************************************************************
node1                      : ok=332  changed=125  unreachable=0    failed=0

On node 1, stop pcap replay, bro and yaf

# monit summary
# monit stop  pcap-service  pcap-replay  pcap-parser bro   bro-parser  yaf-parser
# pkill -f "/opt/bin/tcpreplay --intf1=eth0 --loop=0 /opt/pcap-replay/example.pcap"

Play with snort logs

# ps faux |grep -B 1 "alert.csv"
flume    25260  0.6  0.7 3580532 114676 ?      Sl   17:14   0:11 /usr/jdk64/jdk1.8.0_77/bin/java -Xmx20m  ...org.apache.flume.node.Application --conf-file /usr/hdp/2.4.3.0-227/flume/conf/flume-snort.conf --name snort
flume    26567  0.0  0.0 100952   660 ?        S    17:14   0:00  \_ tail -F /var/log/snort/alert.csv

# curl http://node1:9200/_cat/indices?v
health status index   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   .kibana   5   1         34            0     73.4kb         73.4kb

No snort index .. Possible issue: Mysql did not start.

# cat /var/log/storm/enrichment-4-1476717306-worker-6703.log
...
2016-10-17 21:17:54.247 o.a.m.e.a.j.JdbcAdapter [ERROR] [Metron] JDBC connection failed.... 
...
# service mysql status
# service mysql start

Restart the topo

# storm kill enrichment
# /usr/metron/0.2.1BETA/bin/start_enrichment_topology.sh
OR
# monit restart enrichment

Oh, a new index :)

$ curl http://node1:9200/_cat/indices?v

http://node1:5000/app/kibana#/dashboard/Metron-Dashboard

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment