Syslog server for Splunk

rsyslog-server.md

Generic syslog building block

The rsyslog config below realises a reusable building block to onboard syslog data, for example into Splunk. It assumes an on-prem enterprise environment and uses the file system as a buffer/queue to decouple syslog senders from a receiver like Splunk Universal Forwarder (UF). This way you can restart Splunk UF without any data loss.

The following four configuration files ensure:

• Reception of syslog into one log file for every source IP address.
• Fitness for a high volume syslog setup by having rsyslog NOT throttle
• Least privilege for Splunk UF by having rsyslog create files with appropriate umask and group
• Retention of all log files for 1 day to prevent availability issues from "disk full" scenarios

1. rsyslog.conf:

#
# /etc/rsyslog.conf: main config file
#

$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imklog   # provides kernel logging support (previously done by rklogd)

#### GLOBAL DIRECTIVES ####

$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
$SystemLogRateLimitInterval 0
$SystemLogRateLimitBurst 0
$umask 0027
$FileGroup splunk

#### RULES ####

*.info;mail.none;authpriv.none;cron.none;local1.none    /var/log/messages
authpriv.*                                              /var/log/secure
mail.*                                                  -/var/log/maillog
cron.*                                                  /var/log/cron
uucp,news.crit                                          /var/log/spooler
local7.*                                                /var/log/boot.log

2. /etc/rsyslog.d/listen.conf:

$SystemLogSocketName /run/systemd/journal/syslog

# Provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514

# Provides TCP syslog reception
$ModLoad imtcp
$InputTCPServerRun 514

3. /etc/rsyslog.d/splunk.conf:

$EscapeControlCharactersOnReceive off

template(name="mydynafile_syslog" type="list"){
  constant(value="/var/log/syslog/")
  property(name="fromhost-ip")
  constant(value="/")
  property(name="fromhost-ip")
  constant(value=".log")
}

if $fromhost-ip != "127.0.0.1" then {
  action (type="omfile" DynaFile="mydynafile_syslog" FileOwner="root" FileGroup="splunk" FileCreateMode="0644" DirOwner="root" DirGroup="splunk" DirCreateMode="0750")
  stop
}

4. /etc/logrotate.d/splunk

/var/log/syslog/*/*.log
{
    daily
    rotate 1
    missingok
    sharedscripts
    postrotate
        /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
    endscript
}