Skip to content

Instantly share code, notes, and snippets.

@chelming
Last active December 9, 2020 15:35
Show Gist options
  • Save chelming/a970041f31f288dd2c16 to your computer and use it in GitHub Desktop.
Save chelming/a970041f31f288dd2c16 to your computer and use it in GitHub Desktop.
NXLOG config that will parse SCCM and IIS logs
## Please set the ROOT to the folder your nxlog was installed into,
## otherwise it will not start.
#define ROOT C:\Program Files\nxlog
define ROOT C:\Program Files (x86)\nxlog
define CERTDIR %ROOT%\cert
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log
<Extension syslog>
Module xm_syslog
</Extension>
<Extension fileop>
Module xm_fileop
</Extension>
## Create the parse rule for IIS logs. You can copy these from the header of the IIS log file.
<Extension w3c>
Module xm_csv
Fields $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $csUser-Agent, $sc-status, $sc-substatus, $sc-win32-status, $time-taken
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ' '
UndefValue -
</Extension>
<Extension json>
Module xm_json
</Extension>
<Extension charconv>
Module xm_charconv
AutodetectCharsets utf-8, euc-jp, utf-16, utf-32, iso8859-2, ucs-2le
</Extension>
##########################################
## NXLOG INTERNAL LOG ##
##########################################
# Nxlog internal logs - Recommended to keep this turned ON so error(s)/Issues with NXLog are reported.
<Input internal>
Module im_internal
Exec $Hostname = hostname_fqdn();
Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000; to_json();
</Input>
##########################################
## FLAT FILES ##
##########################################
## http://nxlog-ce.sourceforge.net/nxlog-docs/en/nxlog-reference-manual.html#im_file
## Input to watch a file of your choosing. After Input, name it whatever you want to describe that NXLog
## is pulling, then add that name to the path in Route 1 after eventlog. Can be separated for filtering diff logs.
## After setting the Message as raw_event this converts the message to UTF-8, drops empty messages
## removes extra whitespace, grabs the file name as LogFile, adds the FQDN, and deletes a useless var
## Pulls all logfiles from the default ITS Log Location
## It is Recommended to LEAVE THIS ENABLED
## Ensure that "ITS_Logs" is specified in the correct Route at the bottom for output
<Input ITS_Logs>
Module im_file
File "C:\\ITS\\Logs\\\\*.log"
SavePos TRUE
Recursive TRUE
Exec $Message = $raw_event;
Exec $Message = convert($Message, "ucs-2le", "utf-8");
Exec if $Message == '' drop();
Exec if $Message =~ s/^\s+//g log_debug("whitespace removed");
Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1;
Exec $Hostname = hostname_fqdn();
Exec delete($SourceModuleType);
Exec $EventTime = $EventReceivedTime;
Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
Exec to_json();
</Input>
## Pulls and parses logfiles generated by SCCM
## It is Recommended to LEAVE THIS DISABLED unless server uses SCCM
## Ensure that if enabled, "SCCM_Logs" is specified in the correct Route at the bottom for output
##<Input SCCM_Logs>
## Module im_file
## File "E:\\Program Files\\SMS_CCM\\\\*.log"
## SavePos TRUE
## Recursive TRUE
## Exec $Message = $raw_event;
## ##Exec $Message = convert($Message, "ucs-2le", "utf-8");
## Exec if $Message == '' drop();
## Exec if $Message =~ s/^\s+//g log_debug("whitespace removed");
## Exec $Raw_Message = $Message;\
## if $Message =~ /(?:G\[)([^\]]+).*(?:component=")(\w*).*(?:context=")(\w*).*(?:file=")([a-z]+.[a-z]+):([0-9]*)"/ \
## $Message=$1 \
## $Component=$2 \
## $Context=$3 \
## $SCCM_file=$4 \
## $SCCM_linenumber=$5;
## Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1;
## Exec $Hostname = hostname_fqdn();
## Exec delete($SourceModuleType);
## Exec $EventTime = $EventReceivedTime;
## Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
## Exec to_json();
##</Input>
## Pulls and parses logfiles generated by SCCM Agent
## It is Recommended to LEAVE THIS ENABLED unless server isnt using SCCM
## Ensure that if enabled, "SCCM_Agent" is specified in the correct Route at the bottom for output
##<Input SCCM_AGENT>
## Module im_file
## File "C:\\Windows\\CCM\\Logs\\\\*.log"
## SavePos TRUE
## Recursive TRUE
## Exec $Message = $raw_event;
## ##Exec $Message = convert($Message, "ucs-2le", "utf-8");
## Exec if $Message == '' drop();
## Exec if $Message =~ s/^\s+//g log_debug("whitespace removed");
## Exec $Raw_Message = $Message;\
## if $Message =~ /(?:G\[)([^\]]+).*(?:component=")(\w*).*(?:context=")(\w*).*(?:file=")([a-z]+.[a-z]+):([0-9]*)"/ \
## $Message=$1 \
## $Component=$2 \
## $Context=$3 \
## $SCCM_file=$4 \
## $SCCM_linenumber=$5;
## Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1;
## Exec $Hostname = hostname_fqdn();
## Exec delete($SourceModuleType);
## Exec $EventTime = $EventReceivedTime;
## Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
## Exec to_json();
##</Input>
## Pulls and parses logfiles generated by SCCM Agent Setup Program
## It is Recommended to LEAVE THIS ENABLED unless server isn't using SCCM
## Ensure that if enabled, "SCCM_Agent_Setup" is specified in the correct Route at the bottom for output
##<Input SCCM_AGENT_Setup>
## Module im_file
## File "C:\\Windows\\CCMSetup\\Logs\\\\*.log"
## SavePos TRUE
## Recursive TRUE
## Exec $Message = $raw_event;
## ##Exec $Message = convert($Message, "ucs-2le", "utf-8");
## Exec if $Message == '' drop();
## Exec if $Message =~ s/^\s+//g log_debug("whitespace removed");
## Exec $Raw_Message = $Message;\
## ##breaks sccm logs in the <![LOG[message]LOG]> format down into separate fields
## if $Message =~ /(?:G\[)([^\]]+).*(?:component=")(\w*).*(?:context=")(\w*).*(?:file=")([a-z]+.[a-z]+):([0-9]*)"/ \
## $Message=$1; \
## $Component=$2; \
## $Context=$3; \
## $SCCM_file=$4 \
## $SCCM_linenumber=$5;
## Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1;
## Exec $Hostname = hostname_fqdn();
## Exec delete($SourceModuleType);
## Exec $EventTime = $EventReceivedTime;
## Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
## Exec to_json();
##</Input>
## Pulls all logfiles from the Firewall Logs
## It is Recommended to LEAVE THIS ENABLED, and set the Path as appropriate
## Ensure that "Firewall" is specified in the correct Route at the bottom for output
##<Input Firewall>
## Module im_file
## File "E:\\firewall\\\\*.log"
## SavePos TRUE
## Recursive TRUE
## Exec $Message = $raw_event;
## ##Exec $Message = convert($Message, "ucs-2le", "utf-8");
## Exec if $Message == '' drop();
## Exec if $Message =~ s/^\s+//g log_debug("whitespace removed");
## Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1;
## Exec $Hostname = hostname_fqdn();
## Exec $EventTime = $EventReceivedTime;
## Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
## Exec delete($SourceModuleType);
## Exec to_json();
##</Input>
## Pulls and parses logfiles from IIS
## It is Recommended to LEAVE THIS DISABLED, unless the server uses IIS, and set the Path as appropriate
## Ensure that "IISLogs" is specified in the correct Route at the bottom for output
##<Input IISLogs>
## Module im_file
## File "E:\\IISLogs\\\\*.log"
## SavePos TRUE
## Recursive TRUE
##
##
## Exec $Hostname = hostname_fqdn();
## Exec if $raw_event =~ /^#/ drop() \
## else \
## { \
## w3c->parse_csv(); \
## $EventTime = parsedate($date + " " + $time); \
## $SourceName = "IIS"; \
## $Message = to_json(); \
## }
##</Input>
## Pulls and parses logfiles from SSRS (SQL Services Reporting Services)
## It is Recommended to LEAVE THIS DISABLED, unless the server uses SSRS, and set the Path as appropriate
## Ensure that "SSRSLogs" is specified in the correct Route at the bottom for output
##<Input SSRSLogs>
## Module im_file
## File "F:\\Program Files\\Microsoft SQL Server\\MSRS11.MSSQLSERVER\\Reporting Services\\\\*.log"
## SavePos TRUE
## Recursive TRUE
## Exec $Message = $raw_event;
## Exec $Message = convert($Message, "ucs-2le", "utf-8"); ##converts UTF-16 charset to UTF-8
## Exec if $Message == '' drop(); ##drops blank messages
## Exec if $Message =~ s/^\s+//g log_debug("whitespace removed"); ##removes whitespace at the beginning of a line
## Exec if file_name() =~ /([^\\]+)$/ $LogFile = $1; ##grabs the filename without the full path
## Exec $Hostname = hostname_fqdn(); ##adds FQDN as Hostname field
## Exec delete($SourceModuleType);
## Exec $EventTime = $EventReceivedTime; ##keeps the standard time format
## Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000; ##converts time to ES appropriate time
## Exec to_json();
##</Input>
##########################################
## EVENT LOGS ##
##########################################
## Pulls and parses logfiles from Windows Event Logs
## It is Recommended to LEAVE THIS ENABLED
## Ensure that "eventlog" is specified in the correct Route at the bottom for output
<Input eventlog>
Module im_msvistalog
Query <QueryList>\
<Query Id="0">\
<Select Path="Application">*</Select>\
<Select Path="System">*</Select>\
<Select Path="Security">*</Select>\
</Query>\
</QueryList>
Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000; to_json();
</Input>
## http://help.papertrailapp.com/kb/configuration/configuring-remote-syslog-from-windows
## Encrypted TCP Output
<Output sslout>
Module om_ssl
Host <logstash IP>
Port 5544
CAFile %CERTDIR%\ca.pem
AllowUntrusted FALSE
</Output>
##########################################
## SSL OUTPUT ##
##########################################
## Be sure to add in Output for any custom Output defined above to ensure it is properly sent to the LogStash Server
<Route 1>
Path internal, ITS_Logs, SCCM_Logs, Firewall, IISLogs, SSRSLogs, eventlog => sslout
</Route>
##########################################
## DEBUGGING ##
##########################################
## DEBUG OUTPUT ONLY
## It is recommended to keep this DISABLED unless debugging/testing
## Enabling this (uncommenting) allows an Output to a file (can be specified)
## This file can get EXTREMELY Large very quickly
##<Output OutFile>
## Module om_file
## File "C:\\Test.log"
##</Output>
## DEBUG OUTPUT ONLY
## This Route will output the selected things to the Output File Defined above
## It is recommended to keep this DISABLED unless debugging/testing
## Enabling this (uncommenting) allows an Output to a file (can be specified)
## This file can get EXTREMELY Large very quickly
##<Route 2>
## Path ITS_Logs, Firewall, eventlog => OutFile
##</Route>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment