Logstash configuration for ingesting old IAS logs

ias_history.conf

# Using information from:
# - https://iso.csusb.edu/tools/nps-log-interpreter
# - https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2008-R2-and-2008/dd197432(v=ws.10)?redirectedfrom=MSDN
# - http://www.gnu.org/software/radius/manual/html_node/radius_181.html#SEC300
# - https://discuss.elastic.co/t/can-dissect-use-a-variable-number-of-fields/200952/11

input {
	stdin { }
}

filter {

	mutate {
		gsub => ["message", "\r", ""]
	}

	dissect {
		mapping => {
			"message" => "%{NASIPAddress},%{IASUserName},%{Date},%{Time},%{IASServiceType},%{ServerName},%{values}"
		}
	}

	mutate {
		gsub => ["values", "([^,]+),([^,]+),?", "\1=\2,"]
		gsub => ["values", "\b1\b=", "UserName="]
		gsub => ["values", "\b4\b=", "NASIPAddress="]
		gsub => ["values", "\b5\b=", "NASPort="]
		gsub => ["values", "\b6\b=", "ServiceType="]
		gsub => ["values", "\b7\b=", "FramedProtocol="]
		gsub => ["values", "\b8\b=", "FramedIPAddress="]
		gsub => ["values", "\b9\b=", "FramedIPNetmask="]
		gsub => ["values", "\b10\b=", "FramedRouting="]
		gsub => ["values", "\b11\b=", "FilterID="]
		gsub => ["values", "\b12\b=", "FramedMTU="]
		gsub => ["values", "\b13\b=", "FramedCompression="]
		gsub => ["values", "\b14\b=", "LoginIPHost="]
		gsub => ["values", "\b15\b=", "LoginService="]
		gsub => ["values", "\b16\b=", "LoginTCPPort="]
		gsub => ["values", "\b18\b=", "ReplyMessage="]
		gsub => ["values", "\b19\b=", "CallbackNumber="]
		gsub => ["values", "\b20\b=", "CallbackID="]
		gsub => ["values", "\b22\b=", "FramedRoute="]
		gsub => ["values", "\b23\b=", "FramedIPXNetwork="]
		gsub => ["values", "\b25\b=", "Class="]
		gsub => ["values", "\b26\b=", "VendorSpecific="]
		gsub => ["values", "\b27\b=", "SessionTimeout="]
		gsub => ["values", "\b28\b=", "IdleTimeout="]
		gsub => ["values", "\b29\b=", "TerminationAction="]
		gsub => ["values", "\b30\b=", "CalledStationID="]
		gsub => ["values", "\b31\b=", "CallingStationID="]
		gsub => ["values", "\b32\b=", "NASIdentifier="]
		gsub => ["values", "\b34\b=", "LoginLATService="]
		gsub => ["values", "\b35\b=", "LoginLATNode="]
		gsub => ["values", "\b36\b=", "LoginLATGroup="]
		gsub => ["values", "\b37\b=", "FramedAppleTalkLink="]
		gsub => ["values", "\b38\b=", "FramedAppleTalkNetwork="]
		gsub => ["values", "\b39\b=", "FramedAppleTalkZone="]
		gsub => ["values", "\b40\b=", "AcctStatusType="]
		gsub => ["values", "\b41\b=", "AcctDelayTime="]
		gsub => ["values", "\b42\b=", "AcctInputOctets="]
		gsub => ["values", "\b43\b=", "AcctOutputOctets="]
		gsub => ["values", "\b44\b=", "AcctSessionID="]
		gsub => ["values", "\b45\b=", "AcctAuthentic="]
		gsub => ["values", "\b46\b=", "AcctSessionTime="]
		gsub => ["values", "\b47\b=", "AcctInputPackets="]
		gsub => ["values", "\b48\b=", "AcctOutputPackets="]
		gsub => ["values", "\b49\b=", "AcctTerminateCause="]
		gsub => ["values", "\b50\b=", "AcctMultiSSNID="]
		gsub => ["values", "\b51\b=", "AcctLinkCount="]
		gsub => ["values", "\b55\b=", "EventTimestamp="]
		gsub => ["values", "\b61\b=", "NASPortType="]
		gsub => ["values", "\b62\b=", "PortLimit="]
		gsub => ["values", "\b63\b=", "LoginLATPort="]
		gsub => ["values", "\b64\b=", "TunnelType="]
		gsub => ["values", "\b65\b=", "TunnelMediumType="]
		gsub => ["values", "\b66\b=", "TunnelClientEndpt="]
		gsub => ["values", "\b67\b=", "TunnelServerEndpt="]
		gsub => ["values", "\b68\b=", "AcctTunnelConnection="]
		gsub => ["values", "\b75\b=", "PasswordRetry="]
		gsub => ["values", "\b76\b=", "Prompt="]
		gsub => ["values", "\b77\b=", "ConnectInfo="]
		gsub => ["values", "\b78\b=", "ConfigurationToken="]
		gsub => ["values", "\b81\b=", "TunnelPvtGroupID="]
		gsub => ["values", "\b82\b=", "TunnelAssignmentID="]
		gsub => ["values", "\b83\b=", "TunnelPreference="]
		gsub => ["values", "\b85\b=", "AcctInterimInterval="]
		gsub => ["values", "\b4108\b=", "ClientIPAddress="]
		gsub => ["values", "\b4116\b=", "NASManufacturer="]
		gsub => ["values", "\b4120\b=", "MSCHAPDomain="]
		gsub => ["values", "\b4121\b=", "MSCHAPError="]
		gsub => ["values", "\b4127\b=", "AuthenticationType="]
		gsub => ["values", "\b4128\b=", "ClientFriendlyName="]
		gsub => ["values", "\b4129\b=", "SAMAccountName="]
		gsub => ["values", "\b4130\b=", "FullyQualifiedUserName="]
		gsub => ["values", "\b4132\b=", "EAPFriendlyName="]
		gsub => ["values", "\b4136\b=", "PacketType="]
		gsub => ["values", "\b4142\b=", "ReasonCode="]
		gsub => ["values", "\b4147\b=", "MSRASVendor="]
		gsub => ["values", "\b4148\b=", "MSRASVersion="]
		gsub => ["values", "\b4149\b=", "NPPolicyName="]
		gsub => ["values", "\b4154\b=", "ProxyPolicyName="]
		gsub => ["values", "\b4155\b=", "ProviderType="]
		gsub => ["values", "\b4156\b=", "ProviderName="]
		gsub => ["values", "\b4157\b=", "RemoteServerAddress="]
		gsub => ["values", "\b4159\b=", "MSRASClientName="]
		gsub => ["values", "\b4160\b=", "MSRASClientVersion="]
		add_field => [ "log_timestamp", "%{Date} %{Time}" ]
	}

	date {
			locale => "en"
			match => [ "log_timestamp", "MM/dd/YYYY HH:mm:ss"]
			timezone => "America/Sao_Paulo"
	}

	kv {
		source => "values"
		field_split => ","
		value_split => "="
	}

	mutate {
		remove_field => ["values"]
	}

	translate {
		field => "[PacketType]"
		destination => "[PacketType_desc]"
		dictionary => {
			"1" => "AccessRequest"
			"2" => "AccessAccept"
			"3" => "AccessReject"
			"4" => "AccountingRequest"
			"5" => "AccountingResponse"
			"6" => "AccountingStatus (now Interim Accounting)"
			"7" => "PasswordRequest"
			"8" => "PasswordAck"
			"9" => "PasswordReject"
			"10" => "AccountingMessage"
			"11" => "AccessChallenge"
			"12" => "StatusServer (experimental)"
			"13" => "StatusClient (experimental)"
			"21" => "ResourceFreeRequest"
			"22" => "ResourceFreeResponse"
			"23" => "ResourceQueryRequest"
			"24" => "ResourceQueryResponse"
			"25" => "AlternateResourceReclaimRequest"
			"26" => "NASRebootRequest"
			"27" => "NASRebootResponse"
			"28" => "Reserved"
			"29" => "NextPasscode"
			"30" => "NewPin"
			"31" => "TerminateSession"
			"32" => "PasswordExpired"
			"33" => "EventRequest"
			"34" => "EventResponse"
			"35" => "Unassigned"
			"36" => "Unassigned"
			"37" => "Unassigned"
			"38" => "Unassigned"
			"39" => "Unassigned"
			"40" => "DisconnectRequest"
			"41" => "DisconnectACK"
			"42" => "DisconnectNAK"
			"43" => "CoARequest"
			"44" => "CoAACK"
			"45" => "CoANAK"
			"46" => "Unassigned"
			"47" => "Unassigned"
			"48" => "Unassigned"
			"49" => "Unassigned"
			"50" => "IPAddressAllocate"
			"51" => "IPAddressRelease"
			"52" => "ProtocolError"
			"53" => "Unassigned"
			"54" => "Unassigned"
			"55" => "Unassigned"
			"56" => "Unassigned"
			"57" => "Unassigned"
			"58" => "Unassigned"
			"59" => "Unassigned"
			"60" => "Unassigned"
			"61" => "Unassigned"
			"62" => "Unassigned"
			"63" => "Unassigned"
			"64" => "Unassigned"
			"65" => "Unassigned"
			"66" => "Unassigned"
			"67" => "Unassigned"
			"68" => "Unassigned"
			"69" => "Unassigned"
			"70" => "Unassigned"
			"71" => "Unassigned"
			"72" => "Unassigned"
			"73" => "Unassigned"
			"74" => "Unassigned"
			"75" => "Unassigned"
			"76" => "Unassigned"
			"77" => "Unassigned"
			"78" => "Unassigned"
			"79" => "Unassigned"
			"80" => "Unassigned"
			"81" => "Unassigned"
			"82" => "Unassigned"
			"83" => "Unassigned"
			"84" => "Unassigned"
			"85" => "Unassigned"
			"86" => "Unassigned"
			"87" => "Unassigned"
			"88" => "Unassigned"
			"89" => "Unassigned"
			"90" => "Unassigned"
			"91" => "Unassigned"
			"92" => "Unassigned"
			"93" => "Unassigned"
			"94" => "Unassigned"
			"95" => "Unassigned"
			"96" => "Unassigned"
			"97" => "Unassigned"
			"98" => "Unassigned"
			"99" => "Unassigned"
			"100" => "Unassigned"
			"101" => "Unassigned"
			"102" => "Unassigned"
			"103" => "Unassigned"
			"104" => "Unassigned"
			"105" => "Unassigned"
			"106" => "Unassigned"
			"107" => "Unassigned"
			"108" => "Unassigned"
			"109" => "Unassigned"
			"110" => "Unassigned"
			"111" => "Unassigned"
			"112" => "Unassigned"
			"113" => "Unassigned"
			"114" => "Unassigned"
			"115" => "Unassigned"
			"116" => "Unassigned"
			"117" => "Unassigned"
			"118" => "Unassigned"
			"119" => "Unassigned"
			"120" => "Unassigned"
			"121" => "Unassigned"
			"122" => "Unassigned"
			"123" => "Unassigned"
			"124" => "Unassigned"
			"125" => "Unassigned"
			"126" => "Unassigned"
			"127" => "Unassigned"
			"128" => "Unassigned"
			"129" => "Unassigned"
			"130" => "Unassigned"
			"131" => "Unassigned"
			"132" => "Unassigned"
			"133" => "Unassigned"
			"134" => "Unassigned"
			"135" => "Unassigned"
			"136" => "Unassigned"
			"137" => "Unassigned"
			"138" => "Unassigned"
			"139" => "Unassigned"
			"140" => "Unassigned"
			"141" => "Unassigned"
			"142" => "Unassigned"
			"143" => "Unassigned"
			"144" => "Unassigned"
			"145" => "Unassigned"
			"146" => "Unassigned"
			"147" => "Unassigned"
			"148" => "Unassigned"
			"149" => "Unassigned"
			"150" => "Unassigned"
			"151" => "Unassigned"
			"152" => "Unassigned"
			"153" => "Unassigned"
			"154" => "Unassigned"
			"155" => "Unassigned"
			"156" => "Unassigned"
			"157" => "Unassigned"
			"158" => "Unassigned"
			"159" => "Unassigned"
			"160" => "Unassigned"
			"161" => "Unassigned"
			"162" => "Unassigned"
			"163" => "Unassigned"
			"164" => "Unassigned"
			"165" => "Unassigned"
			"166" => "Unassigned"
			"167" => "Unassigned"
			"168" => "Unassigned"
			"169" => "Unassigned"
			"170" => "Unassigned"
			"171" => "Unassigned"
			"172" => "Unassigned"
			"173" => "Unassigned"
			"174" => "Unassigned"
			"175" => "Unassigned"
			"176" => "Unassigned"
			"177" => "Unassigned"
			"178" => "Unassigned"
			"179" => "Unassigned"
			"180" => "Unassigned"
			"181" => "Unassigned"
			"182" => "Unassigned"
			"183" => "Unassigned"
			"184" => "Unassigned"
			"185" => "Unassigned"
			"186" => "Unassigned"
			"187" => "Unassigned"
			"188" => "Unassigned"
			"189" => "Unassigned"
			"190" => "Unassigned"
			"191" => "Unassigned"
			"192" => "Unassigned"
			"193" => "Unassigned"
			"194" => "Unassigned"
			"195" => "Unassigned"
			"196" => "Unassigned"
			"197" => "Unassigned"
			"198" => "Unassigned"
			"199" => "Unassigned"
			"200" => "Unassigned"
			"201" => "Unassigned"
			"202" => "Unassigned"
			"203" => "Unassigned"
			"204" => "Unassigned"
			"205" => "Unassigned"
			"206" => "Unassigned"
			"207" => "Unassigned"
			"208" => "Unassigned"
			"209" => "Unassigned"
			"210" => "Unassigned"
			"211" => "Unassigned"
			"212" => "Unassigned"
			"213" => "Unassigned"
			"214" => "Unassigned"
			"215" => "Unassigned"
			"216" => "Unassigned"
			"217" => "Unassigned"
			"218" => "Unassigned"
			"219" => "Unassigned"
			"220" => "Unassigned"
			"221" => "Unassigned"
			"222" => "Unassigned"
			"223" => "Unassigned"
			"224" => "Unassigned"
			"225" => "Unassigned"
			"226" => "Unassigned"
			"227" => "Unassigned"
			"228" => "Unassigned"
			"229" => "Unassigned"
			"230" => "Unassigned"
			"231" => "Unassigned"
			"232" => "Unassigned"
			"233" => "Unassigned"
			"234" => "Unassigned"
			"235" => "Unassigned"
			"236" => "Unassigned"
			"237" => "Unassigned"
			"238" => "Unassigned"
			"239" => "Unassigned"
			"240" => "Unassigned"
			"241" => "Unassigned"
			"242" => "Unassigned"
			"243" => "Unassigned"
			"244" => "Unassigned"
			"245" => "Unassigned"
			"246" => "Unassigned"
			"247" => "Unassigned"
			"248" => "Unassigned"
			"249" => "Unassigned"
			"250" => "Experimental Use"
			"251" => "Experimental Use"
			"252" => "Experimental Use"
			"253" => "Experimental Use"
			"254" => "Reserved"
			"255" => "Reserved"
		}
	}

	translate {
		field => "[ServiceType]"
		destination => "[ServiceType_desc]"
		dictionary => {
			"1" => "LoginUser"
			"2" => "FramedUser"
			"3" => "CallbackLoginUser"
			"4" => "CallbackFramedUser"
			"5" => "OutboundUser"
			"6" => "AdministrativeUser"
			"7" => "NASPromptUser"
			"8" => "AuthenticateOnly"
			"10" => "CallCheck"
		}
	}

	translate {
		field => "[ReasonCode]"
		destination => "[ReasonCode_desc]"
		dictionary => {
			"0" => "SUCCESS"
			"1" => "INTERNAL_ERROR"
			"2" => "ACCESS_DENIED"
			"3" => "MALFORMED_REQUEST"
			"4" => "GLOBAL_CATALOG_UNAVAILABLE"
			"5" => "DOMAIN_UNAVAILABLE"
			"6" => "SERVER_UNAVAILABLE"
			"7" => "NO_SUCH_DOMAIN"
			"8" => "NO_SUCH_USER"
			"9" => "EXTENSION_DISCARD"
			"16" => "AUTH_FAILURE"
			"17" => "CHANGE_PASSWORD_FAILURE"
			"18" => "UNSUPPORTED_AUTH_TYPE"
			"19" => "NO_CLEARTEXT_PASSWORD"
			"20" => "LM_NOT_ALLOWED"
			"21" => "EXTENSION_REJECT"
			"22" => "EAP_NEGOTIATION_FAILED"
			"23" => "UNEXPECTED_EAP_ERROR"
			"32" => "LOCAL_USERS_ONLY"
			"33" => "PASSWORD_MUST_CHANGE"
			"34" => "ACCOUNT_DISABLED"
			"35" => "ACCOUNT_EXPIRED"
			"36" => "ACCOUNT_LOCKED_OUT"
			"37" => "INVALID_LOGON_HOURS"
			"38" => "ACCOUNT_RESTRICTION"
			"48" => "NO_POLICY_MATCH"
			"49" => "NO_CONNECTION_REQUEST_POLICY_MATCH"
			"64" => "DIALIN_LOCKED_OUT"
			"65" => "DIALIN_DISABLED"
			"66" => "INVALID_AUTH_TYPE"
			"67" => "INVALID_CALLING_STATION"
			"68" => "INVALID_DIALIN_HOURS"
			"69" => "INVALID_CALLED_STATION"
			"70" => "INVALID_PORT_TYPE"
			"71" => "DIALIN_RESTRICTION"
			"72" => "CPW_NOT_ALLOWED"
			"73" => "INVALID_CERT_EKU"
			"80" => "NO_RECORD"
			"96" => "SESSION_TIMEOUT"
			"97" => "UNEXPECTED_REQUEST"
			"112" => "PROXY_REJECT"
			"113" => "PROXY_UNKNOWN_GROUP"
			"114" => "PROXY_UNKNOWN_SERVER"
			"115" => "PROXY_PACKET_TOO_LONG"
			"116" => "PROXY_SEND_ERROR"
			"117" => "PROXY_TIMEOUT"
			"118" => "PROXY_MALFORMED_RESPONSE"
			"256" => "CRYPT_E_REVOKED"
			"257" => "CRYPT_E_NO_REVOCATION_DLL"
			"258" => "CRYPT_E_NO_REVOCATION_CHECK"
			"259" => "CRYPT_E_REVOCATION_OFFLINE"
			"260" => "SEC_E_MESSAGE_ALTERED"
			"261" => "SEC_E_NO_AUTHENTICATING_AUTHORITY"
			"262" => "SEC_E_INCOMPLETE_MESSAGE"
			"263" => "SEC_E_INCOMPLETE_CREDENTIALS"
			"264" => "SEC_E_TIME_SKEW"
			"265" => "SEC_E_UNTRUSTED_ROOT"
			"266" => "SEC_E_ILLEGAL_MESSAGE"
			"267" => "SEC_E_CERT_WRONG_USAGE"
			"268" => "SEC_E_CERT_EXPIRED"
			"269" => "SEC_E_ALGORITHM_MISMATCH"
			"270" => "SEC_E_SMARTCARD_LOGON_REQUIRED"
			"271" => "SEC_E_SHUTDOWN_IN_PROGRESS"
			"272" => "SEC_E_MULTIPLE_ACCOUNTS"
			"273" => "TRUST_E_PROVIDER_UNKNOWN"
			"274" => "TRUST_E_ACTION_UNKNOWN"
			"275" => "TRUST_E_SUBJECT_FORM_UNKNOWN"
			"276" => "TRUST_E_SUBJECT_NOT_TRUSTED"
			"277" => "TRUST_E_NOSIGNATURE"
			"278" => "CERT_E_EXPIRED"
			"279" => "CERT_E_VALIDITYPERIODNESTING"
			"280" => "CERT_E_ROLE"
			"281" => "CERT_E_PATHLENCONST"
			"282" => "CERT_E_CRITICAL"
			"283" => "CERT_E_PURPOSE"
			"284" => "CERT_E_ISSUERCHAINING"
			"285" => "CERT_E_MALFORMED"
			"286" => "CERT_E_UNTRUSTEDROOT"
			"287" => "CERT_E_CHAINING"
			"288" => "TRUST_E_FAIL"
			"289" => "CERT_E_REVOKED"
			"290" => "CERT_E_UNTRUSTEDTESTROOT"
			"291" => "CERT_E_REVOCATION_FAILURE"
			"292" => "CERT_E_CN_NO_MATCH"
			"293" => "CERT_E_WRONG_USAGE"
			"294" => "TRUST_E_EXPLICIT_DISTRUST"
			"295" => "CERT_E_UNTRUSTEDCA"
			"296" => "CERT_E_INVALID_POLICY"
			"297" => "CERT_E_INVALID_NAME"
			"298" => "SEC_E_PKINIT_NAME_MISMATCH"
			"299" => "SEC_E_OUT_OF_SEQUENCE"
			"300" => "SEC_E_NO_CREDENTIALS"
		}
	}

	translate {
		field => "[AcctStatusType]"
		destination => "[AcctStatusType_desc]"
		dictionary => {
			"1" => "Start"
			"2" => "Stop"
			"3" => "InterimUpdate"
			"4" => "Unassigned"
			"5" => "Unassigned"
			"6" => "Unassigned"
			"7" => "AccountingOn"
			"8" => "AccountingOff"
			"9" => "TunnelStart"
			"10" => "TunnelStop"
			"11" => "TunnelReject"
			"12" => "TunnelLinkStart"
			"13" => "TunnelLinkStop"
			"14" => "TunnelLinkReject"
			"15" => "Failed"
		}
	}

	translate {
		field => "[AcctTerminateCause]"
		destination => "[AcctTerminateCause_desc]"
		dictionary => {
			"1" => "User Request"
			"2" => "Lost Carrier"
			"3" => "Lost Service"
			"4" => "Idle Timeout"
			"5" => "Session Timeout"
			"6" => "Admin Reset"
			"7" => "Admin Reboot"
			"8" => "Port Error"
			"9" => "NAS Error"
			"10" => "NAS Request"
			"11" => "NAS Reboot"
			"12" => "Port Unneeded"
			"13" => "Port Preempted"
			"14" => "Port Suspended"
			"15" => "Service Unavailable"
			"16" => "Callback"
			"17" => "User Error"
			"18" => "Host Request"
			"19" => "Supplicant Restart"
			"20" => "Reauthentication Failure"
			"21" => "Port Reinitialized"
			"22" => "Port Administratively Disabled"
			"23" => "Lost Power	[Ramprasad_Golla]"
		}
	}

	translate {
		field => "[AcctAuthentic]"
		destination => "[AcctAuthentic_desc]"
		dictionary => {
			"1" => "RADIUS"
			"2" => "Local"
			"3" => "Remote"
		}
	}

	translate {
		field => "[TerminationAction]"
		destination => "[TerminationAction_desc]"
		dictionary => {
			"0" => "Default"
			"1" => "RADIUS-Request"
		}
	}

	translate {
		field => "[NASPortType]"
		destination => "[NASPortType_desc]"
		dictionary => {
			"0" => "Async"
			"1" => "Sync"
			"2" => "ISDN"
			"3" => "ISDNV120"
			"4" => "ISDNV110"
		}
	}
}

output {

	elasticsearch{
		hosts => ["<your_ES>"]
		index => "historico_vpn-%{+YYYY.MM.dd}"
	}
	# stdout { codec => rubydebug }
	# stdout { codec => json }
}