Python variant of nfotx.pl created by AlienVault community user @packetinspector

!ossim_plugin_nfotx.md

This is my variant of nfOTX plugin by @PacketInspector. I rewrote the original nfotx.pl to nfotx.py and added check for my own ip reputation data file.

nfotx.cfg

#Variant of nfotx plugin for OSSIM created by AlienVault community user @PacketInspector
#modified for nfotx.py
;; configuration for rsyslog

;;/etc/rsyslog.d/nfotx.conf 
;;if $programname == 'nfotx' then /var/log/nfotx.log
;;& ~



[DEFAULT]
plugin_id=90011

[config]
type=detector
enable=yes

source=log
# TODO: read from more than one file (/var/log/syslog)
location=/var/log/nfotx.log

# create log file if it does not exists,
# otherwise stop processing this plugin
create_file=false

process=nfotx.py
start=yes   ; launch plugin process when agent starts
stop=no     ; shutdown plugin process when agent stops
restart=yes  ; restart plugin process after each interval
restart_interval=180
startup=/usr/local/bin/nfotx.py
shutdown=


[otxmatch]
event_type=event
regexp="\:\d{2}\s+(?P<host>\S+)\snfotx\:\s(?P<date>\S+\s\S+)\..*?(?P<proto>\d+)\s+(?P<src_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\:(?P<src_port>\d{1,5}).*?(?P<dst_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\:(?P<dst_port>\d+).*?\d+.*(?P<packets>\d+)\s+(?P<bytes>\d+)\s+(?P<pps>\d+)\s+(?P<bps>\d+)\s+(?P<bpp>\d+)\s+(?P<flows>\d+)"
plugin_sid=1
device={resolv($host)}
date={normalize_date($date)}
src_ip={$src_ip}
src_port={$src_port}
dst_ip={$dst_ip}
dst_port={$dst_port}
protocol={$proto}
userdata1={$flows}
userdata2={$bytes}
userdata3={$packets}
userdata4={$pps}
userdata5={$bps}

nfotx.py

#!/usr/bin/python

#/usr/local/bin/nfotx.py
#Author esguardian@outlook.com
#Python variant of nfotx.pl created by AlienVault community user @PacketInspector
#Uses nfdump and looks for matches in otx and your own bad ip list
#your own ip list may be in free format, but each ip address must end with '#'
#version 1.0.0 ctreated at 16:58 08.08.2015
#
import os
import datetime
import syslog
import subprocess
import codecs

#Using syslog to make logs
syslog.openlog('nfotx')

#Polling interval.  Usually equal to watchdog interval (in minutes)
pi = datetime.timedelta(minutes=3)
#next string I have use for correcting system error with Moscow timezone. Now it's not necessary
#offset = datetime.timedelta(hours=1)

#Set some vars
#stats file is smaller...
otx_ip_repfile =  '/etc/ossim/server/reputation.data'
my_ip_repfile = '/etc/my_ossim/my_reputation.data'
mycharset = 'cp1251'

#You may want to extend this directory lower to a specific collector.  You probably don't want to run this against netflow from perimeter for instance
nfdir = '/var/cache/nfdump/flows/live'


#Make a polling date for nfdump to check
current_time = datetime.datetime.now()
nfdump_check_time = (current_time - pi).strftime("%Y/%m/%d.%H:%M:%S")
nfdump_check_now = current_time.strftime("%Y/%m/%d.%H:%M:%S")

#Open the OTX DB
with open(otx_ip_repfile, 'r') as f:
    otx_iprepdata = f.read()
f.close()

#Open the MY_REPUTATION DB
my_rep_data = {}
if os.path.isfile(my_ip_repfile):
    with codecs.open(my_ip_repfile, 'r', encoding=mycharset) as f:
        for line in f:
            if '#' in line:
                (ip,rep) = line.strip().split('#')
                my_rep_data[ip] = rep.strip()
    f.close() 


#Build cmd
nf_dump_cmd = '/usr/bin/nfdump -R ' + nfdir + ' -q -N -m -A srcip,dstip -t '+ nfdump_check_time + '-' + nfdump_check_now + ' -o extended'
p = subprocess.Popen (nf_dump_cmd, stdout=subprocess.PIPE, shell=True)
(output,err) = p.communicate()
p_stutus = p.wait()

#Initialize a hash to check for dupes, keep count for first match reference
dupes = {}
i = 0

for line in output.splitlines() :
    fields = line.rstrip().split()
#   Grab destination, only checking one source since flows are bi-directional
    (dst_ip, dst_port) = fields[6].split(':')
#   Skip if checked already
    if dupes.has_key(dst_ip) : 
        continue
    dupes[dst_ip] = i
#   Now search reputation.data
    search_ip = dst_ip + '#'
    if dst_ip in my_rep_data:
        if my_rep_data[dst_ip].lower() != 'false':
            syslog.syslog(line)
    elif search_ip in otx_iprepdata :
        syslog.syslog(line)
    i = i + 1

nfotx.sql

-- NfOTX
-- plugin_id: 90011
-- use #cat nfotx.sql | ossim-db

DELETE FROM plugin WHERE id = "90011";
DELETE FROM plugin_sid where plugin_id = "90011";

INSERT IGNORE INTO plugin (id, type, name, description) VALUES (90011, 1, 'NfOTX', 'Netflow OTX Matcher');

INSERT IGNORE INTO plugin_sid (plugin_id, sid, category_id, class_id, name, priority, reliability) VALUES (90011, 1, NULL, NULL, 'Netflow OTX Match',1, 3);