Alienvault OTX feed generator that scrapes and stores Tor relay nodes

env

OTX_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
OTX_FEED_NAME=TOR-Relay-Nodes
OTX_FEED_PUBLIC=True
OTX_FEED_OWNER=otxusername

otx-tor.py

#
# Copyright (c) 2017 Benjamin Roberts
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

from stem.descriptor.remote import get_consensus
from OTXv2 import OTXv2
from os import getenv
from datetime import datetime, timedelta
import pickle

# initialise OTX vars
OTX_SERVER = getenv("OTX_SERVER", default="https://otx.alienvault.com/")
OTX_API_KEY = getenv("OTX_API_KEY", default=None)
OTX_FEED_NAME = getenv("OTX_FEED_NAME", default="TOR-Relay-Nodes")
OTX_FEED_PUBLIC = bool(getenv("OTX_FEED_PUBLIC", default=False))
OTX_FEED_OWNER = getenv("OTX_FEED_OWNER", default=None)
OTX_FEED_EXPIRE = bool(getenv("OTX_FEED_EXPIRE", default=True))
OTX_INDICATOR_FILE = getenv("OTX_INDICATOR_FILE", default="./onion-otx-feed-storage.pickle")
FILTER_INDICATOR_FIELD = ["published", "expires", "role"]

if OTX_API_KEY is None:
    print("environmental variable OTX_API_KEY must be set, exiting...")
    exit(1)

# Conenct to OTX
otx = OTXv2(OTX_API_KEY, server=OTX_SERVER)
if otx is None:
    print("failed to connect to OTX")

# search for existing pulse
feed_id = None
feed_results = otx.search_pulses(OTX_FEED_NAME)
for feed_result in feed_results["results"]:
    if feed_result["author_name"] == OTX_FEED_OWNER:
        feed_id = feed_result["id"]

if feed_id is None:
    # create feed
    new_feed = otx.create_pulse(name=OTX_FEED_NAME,
                     public=OTX_FEED_PUBLIC,
                     tlp="white",
                     tags=["tor", "relay", "onion"],
                     references=["torproject.org"],
                     indicators=[])
    print("created feed")
    feed_id = new_feed["id"]

print("using feed %s" % (feed_id))

# get_existing_indicators
try:
    with open(OTX_INDICATOR_FILE, "rb") as r:
        indicators = pickle.load(r)
except:
    indicators = {}
print ("%d existing indicators retrieved" % (len(indicators)))

# Load tor consensus
tor_relays = get_consensus().run()
print("%d active relays found" % (len(tor_relays)))

for relay in tor_relays:
    title = "tor relay %s:%s" % (relay.address, relay.or_port)
    description = "%s:%s (%s, %s) last_published: %s" % (relay.address,
                                                       relay.or_port,
                                                       relay.nickname,
                                                       relay.fingerprint,
                                                       relay.published)
    indicators[(relay.address, relay.or_port)] = {
        "type": "IP",
        "indicator": relay.address,
        "role": "tor_relay",
        "description": description,
        "title": title,
        "published": relay.published #store for calculation in next step
    }

# Update the is_active and expires fields
for relay in indicators.values():
    relay["expires"] = (relay["published"] + timedelta(hours=19)) #one hour grace
    relay["is_active"] = (relay["published"]  + timedelta(hours=18)) > datetime.utcnow()

# Send indicators, filtering for ommited fields
print("sending indicators")
otx.replace_pulse_indicators(feed_id, [{k: str(v) for k,v in indicator.items() if k not in FILTER_INDICATOR_FIELD} for indicator in indicators.values()])

print("saving indicators")
with open(OTX_INDICATOR_FILE, "wb") as r:
    pickle.dump(indicators, r)

otx-tor.timer

[Unit]
Description=Run otx-tor script

[Timer]
OnBootSec=15min
OnUnitActiveSec=1h

[Install]
WantedBy=timers.target

requirements.txt

OTXv2==1.2
simplejson==3.11.1
stem==1.5.4

tor-otx.service

[Unit]
Description=AlienVault OTX TOR pulse updater
After=network.target

[Service]
User=user
Group=group
Type=simple
ExecStart=/path/to/virtualenv/python3 otx-tor.py
WorkingDirectory=/path/to/otx-tor
EnvironmentFile=/path/to/otx-tor/env

[Install]
WantedBy=multi-user.target