mongo2log.json

{
    "channels": [
        "amun.events",
        "dionaea.connections",
        "dionaea.capture",
        "glastopf.events",
        "beeswarm.hive",
        "kippo.sessions",
        "conpot.events",
        "snort.alerts",
        "wordpot.events",
        "shockpot.events",
        "p0f.events",
	"suricata.events",
	"elastichoney.events"
    ],
    "log_file": "/tmp/mhn-splunk-from-mongo.log",
    "formatter_name": "splunk"
}

mongo2log.py

#!/opt/hpfeeds-logger/env/bin/python

import json
import hpfeeds
import sys
import logging
from logging.handlers import RotatingFileHandler
from hpfeedslogger.formatters import splunk, arcsight, json_formatter
from hpfeedslogger import processors

FORMATTERS = {
    'splunk': splunk.format,
    'arcsight': arcsight.format,
    'json': json_formatter.format,
}

handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger = logging.getLogger('logger')
logger.setLevel(logging.INFO)
logger.addHandler(handler)

def main():
    if len(sys.argv) < 2:
        logger.error('No config file found. Exiting')
        return 1

    logger.info('Parsing config file: %s', sys.argv[1])

    config = json.load(file(sys.argv[1]))
    # hpfeeds protocol has trouble with unicode, hence the utf-8 encoding here
    channels    = [c.encode('utf-8') for c in config['channels']]
    logfile     = config['log_file']

    processor = processors.HpfeedsMessageProcessor()
    formatter = FORMATTERS.get(config['formatter_name'])
    if not formatter:
        logger.error('Unsupported data log formatter encountered: %s. Exiting.', config['formatter_name'])
        return 1

    handler = RotatingFileHandler(logfile, maxBytes=100*1024*1024, backupCount=3)
    handler.setFormatter(logging.Formatter('%(message)s'))
    data_logger = logging.getLogger('data')
    data_logger.setLevel(logging.INFO)
    data_logger.addHandler(handler)

    logger.info('Writing events to %s', logfile)



    import pymongo
    client = pymongo.MongoClient("localhost", 27017)
    for rec in client.mnemosyne.hpfeed.find({'channel':{'$in': channels}}):
        try:
            payload = json.dumps(rec['payload'])
        except Exception as e:
            print 'Skipping:', rec, ', error:', e
            continue
        for msg in processor.process(rec['ident'], rec['channel'], payload, ignore_errors=True):
            data_logger.info(formatter(msg))


if __name__ == '__main__':
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        logger.error('KeyboardInterrupt encountered, exiting ...')
        sys.exit(0)

usage.md

Install

copy mongo2log.py to /opt/hpfeeds-logger/

cd /opt/hpfeeds-logger
source env/bin/activate
pip install pymongo==2.7.2

Configuration

edit mongo2log.json so contain the channels of the events you want to store and the log_file you want to store them in.

Use

cd /opt/hpfeeds-logger
source env/bin/activate
python mongo2log.py mongo2log.json