Last active
March 20, 2022 10:00
-
-
Save leodido/69faa90d3b150359a14964b89e1f4ed3 to your computer and use it in GitHub Desktop.
Rsyslog configuration to grab syslog messages from journald, convert them to RFC5424 syslog format and send them with octet-counting framing to a syslog receiver
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This only works with the following docker logging drivers currently: journald, json-file, and CRI-O log files | |
global(processInternalMessages="on") | |
global(parser.permitSlashInProgramName="on") | |
global(workDirectory="/var/spool/rsyslog") # default location for work (spool) files | |
# Raise limits within /etc/systemd/journald.conf on the host(s) - ie., RateLimitIntervalSec=30s + RateLimitBurst=1000000 | |
module(load="imjournal" ignorepreviousmessages="on" ratelimit.interval="60" ratelimit.burst="2000000" persiststateinterval="10000" statefile="/var/spool/rsyslog/imjournal.state") | |
module(load="mmutf8fix") | |
module(load="mmkubernetes" | |
tls.cacert="/run/secrets/kubernetes.io/serviceaccount/ca.crt" | |
tokenfile="/run/secrets/kubernetes.io/serviceaccount/token" | |
annotation_match=["."]) | |
# Extracts k8s metadata | |
action(type="mmkubernetes") | |
# Compose k8s namespace and pod name into an app-name only when they are available | |
template(name="k8s_app" type="list") { | |
property(name="!kubernetes!namespace_name") | |
constant(value="/") | |
property(name="!kubernetes!pod_name") | |
} | |
set $!custom_appname = exec_template("k8s_app"); | |
# Otherwise use the default app-name for journal entries not regarding k8s | |
template(name="appname" type="list") { | |
property(name="APP-NAME") | |
} | |
if $!custom_appname == "/" then { | |
set $!custom_appname = exec_template("appname"); | |
} | |
if $!custom_appname startswith "rsyslogd-" then { | |
set $!custom_appname = "rsyslogd"; | |
} | |
# Use the hostname for journal entries not regarding k8s | |
template(name="hostname" type="list") { | |
property(name="!_HOSTNAME") | |
} | |
set $!custom_hostname = exec_template("hostname"); | |
# When empty it's because message does not come from journald but directly from rsyslogd | |
if $!custom_hostname == "" then { | |
set $!custom_hostname = "FROM-RSYSLOGD"; | |
} | |
# Create structured data containing k8s metadata | |
template(name="k8s_cid" type="list") { | |
property(name="!docker!container_id" position.from="1" position.to="12") | |
} | |
set $!custom_cid = exec_template("k8s_cid"); | |
template(name="k8s_nid" type="list") { | |
property(name="!kubernetes!namespace_id" position.from="1" position.to="12") | |
} | |
set $!custom_nid = exec_template("k8s_nid"); | |
template(name="k8s_pid" type="list") { | |
property(name="!kubernetes!pod_id" position.from="1" position.to="12") | |
} | |
set $!custom_pid = exec_template("k8s_pid"); | |
template(name="k8s_component" type="list") { | |
property(name="!kubernetes!labels!component" position.from="1" position.to="32") | |
} | |
set $!custom_component = exec_template("k8s_component"); | |
template(name="k8s_crevision" type="list") { | |
property(name="!kubernetes!labels!controller-revision-hash" position.from="1" position.to="32") | |
} | |
set $!custom_crevision = exec_template("k8s_crevision"); | |
set $!custom_ids = ""; | |
if $!custom_cid != "" then { | |
set $!custom_ids = 'container="' & $!custom_cid & '"'; | |
} | |
if $!custom_nid != "" then { | |
set $!custom_ids = $!custom_ids & ' namespace="' & $!custom_nid & '"'; | |
} | |
if $!custom_pid != "" then { | |
set $!custom_ids = $!custom_ids & ' pod="' & $!custom_pid & '"'; | |
} | |
if $!custom_ids != "" then { | |
set $!custom_ids = "[id " & $!custom_ids & "]"; | |
} | |
set $!custom_labels = ""; | |
if $!custom_component != "" then { | |
set $!custom_labels = 'component="' & $!custom_component & '"'; | |
} | |
if $!custom_crevision != "" then { | |
set $!custom_labels = $!custom_labels & ' controller-revision-hash="' & $!custom_crevision & '"'; | |
} | |
if $!custom_labels != "" then { | |
set $!custom_labels = "[label " & $!custom_labels & "]"; | |
} | |
template(name="c_sddata" type="list") { | |
property(name="!custom_ids" compressspace="on") | |
property(name="!custom_labels" compressspace="on") | |
} | |
template(name="sddata" type="list") { | |
property(name="STRUCTURED-DATA") | |
} | |
if $!custom_labels == "" and $!custom_ids == "" then { | |
set $!custom_sddata = exec_template("sddata"); | |
} else { | |
set $!custom_sddata = exec_template("c_sddata"); | |
} | |
# Compose RFC5424 message | |
template(name="rfc5424" type="list") { | |
constant(value="<") | |
property(name="PRI") | |
constant(value=">1 ") | |
property(name="TIMESTAMP" dateFormat="rfc3339" date.inUTC="on") | |
constant(value=" ") | |
property(name="!custom_hostname" position.from="1" position.to="255" caseConversion="lower") | |
constant(value=" ") | |
property(name="!custom_appname" position.from="1" position.to="48" caseConversion="lower") | |
constant(value=" ") | |
property(name="PROCID" position.from="1" position.to="128") | |
constant(value=" ") | |
property(name="MSGID" position.from="1" position.to="32") | |
constant(value=" ") | |
property(name="!custom_sddata") | |
constant(value=" ") | |
property(name="msg" droplastlf="on") | |
constant(value="\n") | |
} | |
action(type="mmutf8fix") | |
action(type="omfwd" | |
target="127.0.0.1" | |
port="6514" | |
protocol="tcp" | |
tcp_framing="octet-counted" | |
template="rfc5424" | |
queue.type="LinkedList" | |
queue.size="5000000" | |
queue.filename="forwarding" | |
queue.maxdiskspace="1g") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment