Vector config for parsing auditd logs

vector.toml

[sources.audit_raw]
type = "file"
include = [ "/var/log/audit/audit.log" ]
read_from = "beginning"

[transforms.audit_parsed]
type = "remap"
inputs = ["audit_raw"]
source = '''
  # The format is deceptively complex - it looks like key-value pairs but there's actually a few layers of nesting.
  # This parser is based on reverse-engineering auditd.

  # Each line is a key-value-pair string from the kernel, inserted by auditd into another key-value-pair string as an unquoted value.
  # Fortunately there is just enough context to yank out the kernel string from the auditd string and parse them separately.
  # The \x1d byte is AUDIT_INTERP_SEPERATOR. Auditd inserts this at the end of the kernel string and before the trailing "enrichment fields".
  match1 = parse_regex(.message, r'^(?P<auditd_prefix>.*)\bmsg=(?P<kernel>audit\(.*)\x1d(?P<auditd_suffix>.*)$') ?? {}
  . |= parse_key_value(match1.auditd_prefix) ?? {}
  . |= parse_key_value(match1.auditd_suffix) ?? {}
  kernel_message = match1.kernel || ""

  # The kernel message begins with the timestamp and a sequence number.
  match2 = parse_regex(kernel_message, r'^audit\((?P<timestamp>[0-9.]+):(?P<seqno>\d+)\):\s*(?P<rest>.*)$') ?? {}
  ._parsed_timestamp = parse_timestamp(match2.timestamp, "%s%.f") ?? null
  .timestamp = ._parsed_timestamp || now()
  .seqno = parse_int(match2.seqno, 10) ?? null
  kernel_message = match2.rest || kernel_message

  # The rest of the kernel message is a key-value-pair string; however it may contain _another_ embedded key-value-pair string as the value of the 'msg' key.
  # Fortunately this one is delimited by single quotes.
  # Unfortunately parse_key_value doesn't understand single quotes, so we have to yank this embedded string out with a regex.
  match3, err = parse_regex(kernel_message, r'^(?P<prefix>.*)\bmsg=\x27(?P<quoted>[^\x27]*)\x27(?P<suffix>.*)$')
  if err == null {
    . |= parse_key_value(match3.prefix) ?? {}
    . |= parse_key_value(match3.quoted) ?? {}
    . |= parse_key_value(match3.suffix) ?? {}
  } else {
    . |= parse_key_value(kernel_message) ?? {}
  }
'''



[[tests]]
name = "normal"
[[tests.inputs]]
insert_at = "audit_parsed"
type = "log"
[tests.inputs.log_fields]
file = "/var/log/audit/audit.log"
message = """type=LOGIN msg=audit(1651828644.083:1939): pid=32035 uid=0 subj=system_u:system_r:init_t:s0 old-auid=4294967295 auid=1000 tty=(none) old-ses=4294967295 ses=25 res=1\u001dUID="root" OLD-AUID="unset" AUID="ada\""""
[[tests.outputs]]
extract_from = "audit_parsed"
[[tests.outputs.conditions]]
type = "vrl"
source = '''
  assert_eq!(.type, "LOGIN")
  assert_eq!(.timestamp, t'2022-05-06T09:17:24.083Z')
  assert_eq!(.seqno, 1939)
  assert_eq!(.pid, "32035")
  assert_eq!(.uid, "0")
  assert_eq!(.subj, "system_u:system_r:init_t:s0")
  assert_eq!(."old-auid", "4294967295")
  assert_eq!(.auid, "1000")
  assert_eq!(.tty, "(none)")
  assert_eq!(."old-ses", "4294967295")
  assert_eq!(.ses, "25")
  assert_eq!(.res, "1")
  assert_eq!(.UID, "root")
  assert_eq!(."OLD-AUID", "unset")
  assert_eq!(.AUID, "ada")
'''

[[tests]]
name = "nested msg"
[[tests.inputs]]
insert_at = "audit_parsed"
type = "log"
[tests.inputs.log_fields]
file = "/var/log/audit/audit.log"
message = """type=USER_LOGOUT msg=audit(1651746068.966:32475): pid=122482 uid=0 auid=0 ses=146 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login id=0 exe="/usr/sbin/sshd" hostname=? addr=1.2.3.4 terminal=ssh res=success'\u001dUID="root" AUID="root" ID="root\""""
[[tests.outputs]]
extract_from = "audit_parsed"
[[tests.outputs.conditions]]
type = "vrl"
source = '''
  assert_eq!(.type, "USER_LOGOUT")
  assert_eq!(.timestamp, t'2022-05-05T10:21:08.966Z')
  assert_eq!(.seqno, 32475)
  assert_eq!(.pid, "122482")
  assert_eq!(.uid, "0")
  assert_eq!(.auid, "0")
  assert_eq!(.ses, "146")
  assert_eq!(.subj, "system_u:system_r:sshd_t:s0-s0:c0.c1023")
  assert_eq!(.op, "login")
  assert_eq!(.id, "0")
  assert_eq!(.exe, "/usr/sbin/sshd")
  assert_eq!(.hostname, "?")
  assert_eq!(.addr, "1.2.3.4")
  assert_eq!(.terminal, "ssh")
  assert_eq!(.res, "success")
  assert_eq!(.UID, "root")
  assert_eq!(.AUID, "root")
  assert_eq!(.ID, "root")
'''