Skip to content

Instantly share code, notes, and snippets.

@leodido
Last active March 20, 2022 10:00
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save leodido/69faa90d3b150359a14964b89e1f4ed3 to your computer and use it in GitHub Desktop.
Save leodido/69faa90d3b150359a14964b89e1f4ed3 to your computer and use it in GitHub Desktop.
Rsyslog configuration to grab syslog messages from journald, convert them to RFC5424 syslog format and send them with octet-counting framing to a syslog receiver
# This only works with the following docker logging drivers currently: journald, json-file, and CRI-O log files
global(processInternalMessages="on")
global(parser.permitSlashInProgramName="on")
global(workDirectory="/var/spool/rsyslog") # default location for work (spool) files
# Raise limits within /etc/systemd/journald.conf on the host(s) - ie., RateLimitIntervalSec=30s + RateLimitBurst=1000000
module(load="imjournal" ignorepreviousmessages="on" ratelimit.interval="60" ratelimit.burst="2000000" persiststateinterval="10000" statefile="/var/spool/rsyslog/imjournal.state")
module(load="mmutf8fix")
module(load="mmkubernetes"
tls.cacert="/run/secrets/kubernetes.io/serviceaccount/ca.crt"
tokenfile="/run/secrets/kubernetes.io/serviceaccount/token"
annotation_match=["."])
# Extracts k8s metadata
action(type="mmkubernetes")
# Compose k8s namespace and pod name into an app-name only when they are available
template(name="k8s_app" type="list") {
property(name="!kubernetes!namespace_name")
constant(value="/")
property(name="!kubernetes!pod_name")
}
set $!custom_appname = exec_template("k8s_app");
# Otherwise use the default app-name for journal entries not regarding k8s
template(name="appname" type="list") {
property(name="APP-NAME")
}
if $!custom_appname == "/" then {
set $!custom_appname = exec_template("appname");
}
if $!custom_appname startswith "rsyslogd-" then {
set $!custom_appname = "rsyslogd";
}
# Use the hostname for journal entries not regarding k8s
template(name="hostname" type="list") {
property(name="!_HOSTNAME")
}
set $!custom_hostname = exec_template("hostname");
# When empty it's because message does not come from journald but directly from rsyslogd
if $!custom_hostname == "" then {
set $!custom_hostname = "FROM-RSYSLOGD";
}
# Create structured data containing k8s metadata
template(name="k8s_cid" type="list") {
property(name="!docker!container_id" position.from="1" position.to="12")
}
set $!custom_cid = exec_template("k8s_cid");
template(name="k8s_nid" type="list") {
property(name="!kubernetes!namespace_id" position.from="1" position.to="12")
}
set $!custom_nid = exec_template("k8s_nid");
template(name="k8s_pid" type="list") {
property(name="!kubernetes!pod_id" position.from="1" position.to="12")
}
set $!custom_pid = exec_template("k8s_pid");
template(name="k8s_component" type="list") {
property(name="!kubernetes!labels!component" position.from="1" position.to="32")
}
set $!custom_component = exec_template("k8s_component");
template(name="k8s_crevision" type="list") {
property(name="!kubernetes!labels!controller-revision-hash" position.from="1" position.to="32")
}
set $!custom_crevision = exec_template("k8s_crevision");
set $!custom_ids = "";
if $!custom_cid != "" then {
set $!custom_ids = 'container="' & $!custom_cid & '"';
}
if $!custom_nid != "" then {
set $!custom_ids = $!custom_ids & ' namespace="' & $!custom_nid & '"';
}
if $!custom_pid != "" then {
set $!custom_ids = $!custom_ids & ' pod="' & $!custom_pid & '"';
}
if $!custom_ids != "" then {
set $!custom_ids = "[id " & $!custom_ids & "]";
}
set $!custom_labels = "";
if $!custom_component != "" then {
set $!custom_labels = 'component="' & $!custom_component & '"';
}
if $!custom_crevision != "" then {
set $!custom_labels = $!custom_labels & ' controller-revision-hash="' & $!custom_crevision & '"';
}
if $!custom_labels != "" then {
set $!custom_labels = "[label " & $!custom_labels & "]";
}
template(name="c_sddata" type="list") {
property(name="!custom_ids" compressspace="on")
property(name="!custom_labels" compressspace="on")
}
template(name="sddata" type="list") {
property(name="STRUCTURED-DATA")
}
if $!custom_labels == "" and $!custom_ids == "" then {
set $!custom_sddata = exec_template("sddata");
} else {
set $!custom_sddata = exec_template("c_sddata");
}
# Compose RFC5424 message
template(name="rfc5424" type="list") {
constant(value="<")
property(name="PRI")
constant(value=">1 ")
property(name="TIMESTAMP" dateFormat="rfc3339" date.inUTC="on")
constant(value=" ")
property(name="!custom_hostname" position.from="1" position.to="255" caseConversion="lower")
constant(value=" ")
property(name="!custom_appname" position.from="1" position.to="48" caseConversion="lower")
constant(value=" ")
property(name="PROCID" position.from="1" position.to="128")
constant(value=" ")
property(name="MSGID" position.from="1" position.to="32")
constant(value=" ")
property(name="!custom_sddata")
constant(value=" ")
property(name="msg" droplastlf="on")
constant(value="\n")
}
action(type="mmutf8fix")
action(type="omfwd"
target="127.0.0.1"
port="6514"
protocol="tcp"
tcp_framing="octet-counted"
template="rfc5424"
queue.type="LinkedList"
queue.size="5000000"
queue.filename="forwarding"
queue.maxdiskspace="1g")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment