very contrived example of caching matcher

example.md

from datetime import datetime, timedelta

from stream_alert.shared.rule import rule

import boto3
from botocore.exceptions import ClientError


class CachingMatcher(object):

    # Store a default date
    _LAST_REFRESH = datetime(year=1970, month=1, day=1)
    CACHED_VALUE = None

    @classmethod
    def value(cls, minutes=10):

        now = datetime.utcnow()
        refresh_delta = timedelta(minutes=minutes)

        needs_refresh = (cls._LAST_REFRESH + refresh_delta < now) or cls.CACHED_VALUE is None
        if not needs_refresh:
            # no refresh needed so return cached value
            return cls.CACHED_VALUE

        # Refreshing will now occur

        cls.CACHED_VALUE = cls.update_value()

        cls._LAST_REFRESH = now

        return cls.CACHED_VALUE

    @classmethod
    def update_value(cls):
        raise NotImplementedError('implement the update_value classmethod on subclass')


class AccountIDsMatcher(CachingMatcher):

    @classmethod
    def update_value(cls):

        account_ids = set()

        client = boto3.client('organizations')

        args = {}
        next_token = True

        while next_token:
            if next_token and next_token is not True:
                args['NextToken'] = next_token
            try:
                response = client.list_accounts(**args)
                for account in response['Accounts']:
                    account_ids.add(account['Id'])

                next_token = response.get('NextToken')
            except ClientError as err:
                print err
                return account_ids

        return account_ids

    @classmethod
    def unknown_account(cls, rec):
        # refresh every 60 minutes vs default of every 10
        return rec.get('account') not in cls.value(minutes=60)

###
# Basic rule using matcher directly
###
@rule(logs=['cloudwatch:cloudtrail'],
      matchers=[AccountIDsMatcher.unknown_account])
def is_unknown_account_id(rec):
    return True

###
# Basic rule using cached value in rule
###
@rule(logs=['cloudwatch:cloudtrail'])
def is_unknown_account_id(rec):
    return rec.get('account') not in AccountIDsMatcher.value(minutes=60)

I would shy away from the terminology of "Caching" if we're intending to use this as a configuration store.

Could keep with the naming convention and call it like StreamConfig or something

I would DEFINITELY avoid making an HTTP call to the outside world in the middle of a rules execution. Better is to have it pull from a central store and have a system that's separate from StreamAlert's pipeline to run the above logic and periodically update. It reduces the chance of cache inconsistency issues or stampeding.