output.txt

# HELP oom_kills The number of OOM kills since boot by process name
# TYPE oom_kills counter
oom_kills{procname="python"} 21

# HELP oom_kills_total The total number of OOM kills since boot
# TYPE oom_kills_total counter
oom_kills_total 21

# HELP oom_triggers The number of OOM trigger events since boot by process name
# TYPE oom_triggers counter
oom_triggers{procname="gnatsd"} 4
oom_triggers{procname="in:imuxsock"} 1
oom_triggers{procname="jbd2/vda1-8"} 4
oom_triggers{procname="nats_exporter"} 3
oom_triggers{procname="node_exporter"} 5
oom_triggers{procname="python"} 2
oom_triggers{procname="systemd-journal"} 2

HELP oom_triggers_total The total number of OOM trigger events since boot
TYPE oom_triggers_total counter
oom_triggers_total 21

prometheus_oom_counters.pl

#!/usr/bin/env perl
use strict;
use warnings;
use feature 'say';
# TODO: dmesg discovery
# * /var/log/messages
# * /var/log/syslog
# * etc
my $dmesg_bin = "/bin/dmesg";

my %triggers;
my %ooms;
my @dmesg_out = `$dmesg_bin`;
for (@dmesg_out) {
    if (/^(?:\[[^]]+\]) (\S+) invoked oom-killer/) {
        $triggers{$1}++;
        next;
    }
    if (/^(?:\[[^]]+\]) Out of memory: Kill process \S+ \(([^)]+)\)/) {
        $ooms{$1}++;
        next;
    }
}

my @ooms_data;
my $ooms_total = 0;
for my $k (sort keys %ooms) {
    push @ooms_data, {labels => {procname => $k}, value => $ooms{$k}};
    $ooms_total += $ooms{$k};
}
#push @ooms_data, {labels => {procname => "*"}, value => $ooms_total};
say prom_metric("oom_kills", "counter",
    "The number of OOM kills since boot by process name",
    \@ooms_data);
say prom_metric("oom_kills_total", "counter",
    "The total number of OOM kills since boot",
    [{value => $ooms_total}]);

my @triggers_data;
my $triggers_total = 0;
for my $k (sort keys %triggers) {
    push @triggers_data, {labels => {procname => $k}, value => $triggers{$k}};
    $triggers_total += $triggers{$k};
}
#push @triggers_data, {labels => {procname => "*"}, value => $triggers_total};
say prom_metric("oom_triggers", "counter",
    "The number of OOM trigger events since boot by process name",
    \@triggers_data);
say prom_metric("oom_triggers_total", "counter",
    "The total number of OOM trigger events since boot",
    [{value => $triggers_total}]);
exit;

sub prom_metric{
    my ($name, $type, $help, $metrics) = @_;
    $metrics //= [];
    my @lines;
    for my $data (@$metrics) {
        my @labels = map {"$_=\"$data->{labels}{$_}\"" }
                     keys %{ $data->{labels}//{} };
        push @lines, "$name"
            . (@labels?"{":"") . join(", ",@labels) . (@labels?"}":"") . " $data->{value}";
    }
    return join("\n",
        "# HELP $name $help",
        "# TYPE $name $type",
        @lines,
        "");
}