Measuring log file growth with datadog/dogstream

loggrowth.py

from datetime import datetime
from dogstream import common
import logging
import os
import time

METRIC_NAME_COUNT = "log.growth_lines"
METRIC_NAME_CHARS = "log.growth_chars"

EOL_SIZE = len(os.linesep)

def utc_now():
    return int(time.time())

class LogGrowth(object):
    """
    A stateful dogstream parser that tracks the growth (in lines and chars) of a log file,
    allowing log file growth to be reported as a metric (especially useful in the context
    of log file rotation/truncation, where the 'dir' check is insufficient to measure this).

    Implemented as a class based dogstream log parser - i.e. a class providing a 'parse_line' method.
    """

    def __init__(self, logger, log_path, config, user_args):
        self.logger = logger
        self.log_path = log_path
        self.config = config
        self.user_args = user_args

        self._emit_interval = 5
        self._metric_attrs = { 'metric_type': 'counter', 'device_name': log_path }

        self._reset()

    def parse_line(self, line):
        """ dogstream log tailing hook
            Count lines/chars, and emit 'counter' style metrics.

            Metrics are emitted at intervals, not on every line, to avoid dogstream
            having to aggregate per-line metrics on high volume logs.
        """

        self._line_count = self._line_count + 1
        self._char_count = self._char_count + len(line) + EOL_SIZE

        now = utc_now()
        if now > (self._last_emitted + self._emit_interval):
            metrics = [ (METRIC_NAME_COUNT, now, self._line_count, self._metric_attrs),
                        (METRIC_NAME_CHARS, now, self._char_count, self._metric_attrs) ]

            if self.logger.isEnabledFor(logging.DEBUG):
                self.logger.debug("Emitting %s", metrics)
            self._reset()
            return metrics
        else:
            return None

    def _reset(self):
        self._line_count = 0L
        self._char_count = 0L
        self._last_emitted = utc_now()