Prometheus exporter for stats *about* your code!

very_meta_exporter

#!/usr/bin/env python3

# TODO publish this separately
#
# Needs packages:
#  - ag (package probably named something like silversearcher)
#  - cloc
# Uses prometheus_client Python library -- install with pip install --user prometheus_client, or use pipenv
#
# use with something like:
#   * *    *   *   *   cd /home/kamila/scion-p4netfpga && /home/kamila/.local/bin/pipenv run ./very_meta_exporter.py && mv ./very_meta_metrics.prom /var/metrics/
# in crontab, and configure node_exporter's textfile collector to collect from /var/metrics

import csv
import subprocess
import sys

from prometheus_client import CollectorRegistry, Gauge, write_to_textfile

PREFIX='very_meta_'
REGISTRY = CollectorRegistry()


g_matches = Gauge(PREFIX+'match_count',
                  'Number of matches of the given pattern',
                  ['pattern'],
                  registry=REGISTRY)
g_matches_files = Gauge(PREFIX+'match_file_count',
                        'Number of files containing the given pattern',
                        ['pattern'],
                        registry=REGISTRY)

def count_matches(pattern, paths, opts=None):
    if opts == None: opts = []
    out = subprocess.check_output(
	# ag version on ubuntu doesn't support --stats-only :-/
        ['ag', '--stats'] + opts + [pattern, '--'] + paths,
        timeout=10,
        universal_newlines=True,
    )
    lines = out.split('\n')
    num_matches = lines[-6].split()[0]
    num_files   = lines[-5].split()[0]
    return num_matches, num_files

def export_matches(pattern, paths, opts=None):
    num_matches, num_files = count_matches(pattern, paths, opts)
    g_matches.labels(pattern).set(num_matches)
    g_matches_files.labels(pattern).set(num_files)


g_loc = Gauge(PREFIX+'loc_count',
              'Lines of code/comments/whitespace, by language',
              ['lang', 'type'],
              registry=REGISTRY)

g_files = Gauge(PREFIX+'file_count',
                'Number of files by language',
                ['lang'],
                registry=REGISTRY)

def count_loc(paths, opts=None):
    if opts == None: opts = []
    out = subprocess.check_output(
        ['cloc', '--quiet', '--csv'] + opts + ['--'] + paths,
        timeout=10,
        universal_newlines=True,
    )
    lines = out.split('\n')
    return csv.DictReader(lines[1:])

def export_loc(paths, opts):
    TYPES = ('blank', 'comment', 'code')
    for row in count_loc(paths, opts):
        lang = row['language']
        g_files.labels(lang).set(row['files'])
        for t in TYPES:
            g_loc.labels(lang, t).set(row[t])

def main():
    # TODO this should be configurable -- fix when published
    export_matches('TODO',  ['.'], ['--ignore', 'doc', '--ignore', 'hw', '--ignore', 'work'])
    export_matches('SDNet', ['.'], ['--ignore', 'doc', '--ignore', 'hw', '--ignore', 'work'])
    export_loc(['.'], ['--force-lang=c,p4', '--exclude-dir', 'doc,hw,work', '--exclude-ext', 'inc,tcl'])
    write_to_textfile('{}metrics.prom'.format(PREFIX), REGISTRY)

if __name__ == '__main__':
    main()