Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
# This configuration is for rsyslog from RHEL7
module(load="mmjsonparse")
module(load="mmnormalize")
#module(load="omelasticsearch")
#module(load="imfile" mode="inotify")
action(type="mmjsonparse")
lookup_table(name="prio_to_level" file="/etc/rsyslog.d/prio_to_level.json")
lookup_table(name="normalize_level" file="/etc/rsyslog.d/normalize_level.json")
# Process fields from foreman (logger-journald rubygem)
if strlen($!FOREMAN_LOGGER) > 0 then {
set $!foreman!logger = $!FOREMAN_LOGGER;
unset $!FOREMAN_LOGGER;
if strlen($!USER_LOGIN) > 0 then {
set $!foreman!user_login = $!USER_LOGIN;
}
unset $!USER_LOGIN;
if strlen($!USER_ADMIN) > 0 then {
set $!foreman!user_= $!USER_ADMIN;
}
unset $!USER_ADMIN;
if strlen($!ORG_ID) > 0 then {
set $!foreman!org_id = $!ORG_ID;
}
unset $!ORG_ID;
if strlen($!LOC_ID) > 0 then {
set $!foreman!loc_id = $!LOC_ID;
}
unset $!LOC_ID;
if strlen($!REMOTE_IP) > 0 then {
set $!foreman!remote_ip = $!REMOTE_IP;
}
unset $!REMOTE_IP;
if strlen($!REQUEST) > 0 then {
set $!foreman!request = $!REQUEST;
}
unset $!REQUEST;
if strlen($!SESSION) > 0 then {
set $!foreman!session = $!SESSION;
}
unset $!SESSION;
if strlen($!EXCEPTION_MESSAGE) > 0 then {
set $!foreman!exception!message = $!EXCEPTION_MESSAGE;
}
unset $!EXCEPTION_MESSAGE;
if strlen($!EXCEPTION_BACKTRACE) > 0 then {
set $!foreman!exception!backtrace = $!EXCEPTION_BACKTRACE;
}
unset $!EXCEPTION_BACKTRACE;
if strlen($!TEMPLATE_NAME) > 0 then {
set $!foreman!template!name = $!TEMPLATE_NAME;
}
unset $!TEMPLATE_NAME;
if strlen($!TEMPLATE_CONTEXT) > 0 then {
set $!foreman!template!context = $!TEMPLATE_CONTEXT;
}
unset $!TEMPLATE_CONTEXT;
if strlen($!TEMPLATE_DIGEST) > 0 then {
set $!foreman!template!digest = $!TEMPLATE_DIGEST;
}
unset $!TEMPLATE_DIGEST;
if strlen($!TEMPLATE_HOST_NAME) > 0 then {
set $!foreman!template!host_name = $!TEMPLATE_HOST_NAME;
}
unset $!TEMPLATE_HOST_NAME;
if strlen($!TEMPLATE_HOST_ID) > 0 then {
set $!foreman!template!host_id = $!TEMPLATE_HOST_ID;
}
unset $!TEMPLATE_HOST_ID;
}
# The rest is taken mostly from here (with slight changes):
# https://github.com/openshift/origin-aggregated-logging/tree/master/hack/testing/rsyslog
template(name="cnvt_to_viaq_timestamp" type="list") {
property(name="TIMESTAMP" dateFormat="rfc3339")
}
if strlen($!_MACHINE_ID) > 0 then {
# convert from imjournal to viaq systemd format
# https://github.com/ViaQ/elasticsearch-templates/blob/master/namespaces/systemd.yml
set $!systemd!t!MACHINE_ID = $!_MACHINE_ID;
unset $!_MACHINE_ID;
if strlen($!CODE_FILE) > 0 then {
set $!systemd!u!CODE_FILE = $!CODE_FILE;
}
unset $!CODE_FILE;
if strlen($!CODE_FUNCTION) > 0 then {
set $!systemd!u!CODE_FUNCTION = $!CODE_FUNCTION;
}
unset $!CODE_FUNCTION;
if strlen($!CODE_LINE) > 0 then {
set $!systemd!u!CODE_LINE = $!CODE_LINE;
}
unset $!CODE_LINE;
if strlen($!ERRNO) > 0 then {
set $!systemd!u!ERRNO = $!ERRNO;
}
unset $!ERRNO;
if strlen($!MESSAGE_ID) > 0 then {
set $!systemd!u!MESSAGE_ID = $!MESSAGE_ID;
}
unset $!MESSAGE_ID;
if strlen($!RESULT) > 0 then {
set $!systemd!u!RESULT = $!RESULT;
}
unset $!RESULT;
if strlen($!UNIT) > 0 then {
set $!systemd!u!UNIT = $!UNIT;
}
unset $!UNIT;
if strlen($!SYSLOG_FACILITY) > 0 then {
set $!systemd!u!SYSLOG_FACILITY = $!SYSLOG_FACILITY;
}
unset $!SYSLOG_FACILITY;
if strlen($!SYSLOG_IDENTIFIER) > 0 then {
set $!systemd!u!SYSLOG_IDENTIFIER = $!SYSLOG_IDENTIFIER;
}
unset $!SYSLOG_IDENTIFIER;
if strlen($!SYSLOG_PID) > 0 then {
set $!systemd!u!SYSLOG_PID = $!SYSLOG_PID;
}
unset $!SYSLOG_PID;
if strlen($!_AUDIT_LOGINUID) > 0 then {
set $!systemd!t!AUDIT_LOGINUID = $!_AUDIT_LOGINUID;
}
unset $!_AUDIT_LOGINUID;
if strlen($!_AUDIT_SESSION) > 0 then {
set $!systemd!t!AUDIT_SESSION = $!_AUDIT_SESSION;
}
unset $!_AUDIT_SESSION;
if strlen($!_BOOT_ID) > 0 then {
set $!systemd!t!BOOT_ID = $!_BOOT_ID;
}
unset $!_BOOT_ID;
if strlen($!_CAP_EFFECTIVE) > 0 then {
set $!systemd!t!CAP_EFFECTIVE = $!_CAP_EFFECTIVE;
}
unset $!_CAP_EFFECTIVE;
if strlen($!_CMDLINE) > 0 then {
set $!systemd!t!CMDLINE = $!_CMDLINE;
}
unset $!_CMDLINE;
if strlen($!_COMM) > 0 then {
set $!systemd!t!COMM = $!_COMM;
}
unset $!_COMM;
if strlen($!_EXE) > 0 then {
set $!systemd!t!EXE = $!_EXE;
}
unset $!_EXE;
if strlen($!_GID) > 0 then {
set $!systemd!t!GID = $!_GID;
}
unset $!_GID;
if strlen($!_HOSTNAME) > 0 then {
set $!systemd!t!HOSTNAME = $!_HOSTNAME;
}
unset $!_HOSTNAME;
if strlen($!_PID) > 0 then {
set $!systemd!t!PID = $!_PID;
}
unset $!_PID;
if strlen($!_SELINUX_CONTEXT) > 0 then {
set $!systemd!t!SELINUX_CONTEXT = $!_SELINUX_CONTEXT;
}
unset $!_SELINUX_CONTEXT;
if strlen($!_SOURCE_REALTIME_TIMESTAMP) > 0 then {
set $!systemd!t!SOURCE_REALTIME_TIMESTAMP = $!_SOURCE_REALTIME_TIMESTAMP;
}
unset $!_SOURCE_REALTIME_TIMESTAMP;
if strlen($!_SYSTEMD_CGROUP) > 0 then {
set $!systemd!t!SYSTEMD_CGROUP = $!_SYSTEMD_CGROUP;
}
unset $!_SYSTEMD_CGROUP;
if strlen($!_SYSTEMD_OWNER_UID) > 0 then {
set $!systemd!t!SYSTEMD_OWNER_UID = $!_SYSTEMD_OWNER_UID;
}
unset $!_SYSTEMD_OWNER_UID;
if strlen($!_SYSTEMD_SESSION) > 0 then {
set $!systemd!t!SYSTEMD_SESSION = $!_SYSTEMD_SESSION;
}
unset $!_SYSTEMD_SESSION;
if strlen($!_SYSTEMD_SLICE) > 0 then {
set $!systemd!t!SYSTEMD_SLICE = $!_SYSTEMD_SLICE;
}
unset $!_SYSTEMD_SLICE;
if strlen($!_SYSTEMD_UNIT) > 0 then {
set $!systemd!t!SYSTEMD_UNIT = $!_SYSTEMD_UNIT;
}
unset $!_SYSTEMD_UNIT;
if strlen($!_SYSTEMD_USER_UNIT) > 0 then {
set $!systemd!t!SYSTEMD_USER_UNIT = $!_SYSTEMD_USER_UNIT;
}
unset $!_SYSTEMD_USER_UNIT;
if strlen($!_TRANSPORT) > 0 then {
set $!systemd!t!TRANSPORT = $!_TRANSPORT;
}
unset $!_TRANSPORT;
if strlen($!_UID) > 0 then {
set $!systemd!t!UID = $!_UID;
}
unset $!_UID;
if strlen($!_KERNEL_DEVICE) > 0 then {
set $!systemd!k!KERNEL_DEVICE = $!_KERNEL_DEVICE;
}
unset $!_KERNEL_DEVICE;
if strlen($!_KERNEL_SUBSYSTEM) > 0 then {
set $!systemd!k!KERNEL_SUBSYSTEM = $!_KERNEL_SUBSYSTEM;
}
unset $!_KERNEL_SUBSYSTEM;
if strlen($!_UDEV_SYSNAME) > 0 then {
set $!systemd!k!UDEV_SYSNAME = $!_UDEV_SYSNAME;
}
unset $!_UDEV_SYSNAME;
if strlen($!_UDEV_DEVNODE) > 0 then {
set $!systemd!k!UDEV_DEVNODE = $!_UDEV_DEVNODE;
}
unset $!_UDEV_DEVNODE;
if strlen($!_UDEV_DEVLINK) > 0 then {
set $!systemd!k!UDEV_DEVLINK = $!_UDEV_DEVLINK;
}
unset $!_UDEV_DEVLINK;
# these fields require special handling
if strlen($!level) == 0 then {
if strlen($!PRIORITY) > 0 then {
set $!level = lookup("prio_to_level", $!PRIORITY);
}
}
unset $!PRIORITY;
# rsyslog 8.30.0 and later does case insensitive variable name comparison
# which means $!MESSAGE is the same as $!message - HOWEVER - the case
# of the variable name is preserved when outputting, so we need to "normalize"
# to all lower case so that the internal JSON to string conversion will output
# "message" in the outgoing record
if (strlen($!message) == 0) and (strlen($!MESSAGE) == 0) then {
if strlen($!log) > 0 then {
set $!message = $!log;
}
} else {
# see if we're using rsyslog with case sensitivity
if $!message == $!MESSAGE then {
# in case it is really $!MESSAGE - we have to completely unset it, then
# set it in lower case
set $.save_message = $!message;
unset $!message;
set $!message = $.save_message;
unset $.save_message;
} else {
if strlen($!message) == 0 then {
# case sensitive
set $!message = $!MESSAGE;
}
}
}
if strlen($!hostname) == 0 then {
if strlen($!_HOSTNAME) > 0 then {
set $!hostname = $!_HOSTNAME;
} else {
set $!hostname = $hostname;
}
}
unset $!_HOSTNAME;
if strlen($!@timestamp) == 0 then {
# need to figure out how to convert _SOURCE_REALTIME_TIMESTAMP
# in the meantime . . .
set $!@timestamp = exec_template('cnvt_to_viaq_timestamp');
}
unset $!_SOURCE_REALTIME_TIMESTAMP;
unset $!__REALTIME_TIMESTAMP;
# end of block that converts imjournal to viaq format
}
# normalize level
if strlen($!level) > 0 then {
set $.lclevel = tolower($!level);
set $.normlevel = lookup("normalize_level", $.lclevel);
if $.normlevel == "unknown" then {
continue # do nothing - preserve original value
} else {
set $!level = $.normlevel;
}
unset $.lclevel;
unset $.normlevel;
} else {
if $!stream == "stdout" then {
set $!level = "info";
} else {
if $!stream == "stderr" then {
set $!level = "err";
} else {
set $!level = "unknown";
}
}
}
template(name="viaq_template" type="list") {
property(name="$!all-json-plain")
}
# TODO - will send actually to ES in the next step
action(type="omfile" file="/var/log/elastic.log" template="viaq_template")
#action(type="omfile" file="/var/log/elastic-debug.log" template="RSYSLOG_DebugFormat")
@lzap

This comment has been minimized.

Copy link
Owner Author

lzap commented Jul 10, 2018

Example JSON output:

{  
   "level":"info",
   "message":"Completed 200 OK in 5ms (Views: 0.2ms | ActiveRecord: 0.4ms)\n",
   "hostname":"next",
   "systemd":{  
      "t":{  
         "MACHINE_ID":"ce37bff78932646b63dbb85e691a3808",
         "BOOT_ID":"0d88151f05cc4964bbfdceccb9a3ba58",
         "CAP_EFFECTIVE":"0",
         "CMDLINE":"Passenger RackApp: \/usr\/share\/foreman                                      ",
         "COMM":"ruby",
         "EXE":"\/opt\/rh\/rh-ruby24\/root\/usr\/bin\/ruby",
         "GID":"991",
         "HOSTNAME":"next.nat.lan",
         "PID":"15053",
         "SELINUX_CONTEXT":"system_u:system_r:passenger_t:s0",
         "SOURCE_REALTIME_TIMESTAMP":"1531232157689648",
         "SYSTEMD_CGROUP":"\/system.slice\/httpd.service",
         "SYSTEMD_SLICE":"system.slice",
         "SYSTEMD_UNIT":"httpd.service",
         "TRANSPORT":"journal",
         "UID":"993"
      },
      "u":{  
         "SYSLOG_FACILITY":"176",
         "SYSLOG_IDENTIFIER":"foreman"
      }
   },
   "@timestamp":"2018-07-10T15:15:57.689825+01:00",
   "MESSAGE":"Completed 200 OK in 5ms (Views: 0.2ms | ActiveRecord: 0.4ms)\n",
   "foreman":{  
      "logger":"app",
      "user_login":"admin",
      "remote_ip":"192.168.199.1",
      "request":"e9a8e8fd-9fce-41a2-bdc1-a3b706e3396c",
      "session":"61feb0a7-479a-40cd-a4f2-50a365258734"
   }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.