Reformat tcpdump output for logstalgia on macOS

lscap.py

# /usr/bin/env python -u

from colorhash import ColorHash
from pprint import pprint as pp
from pcapng import FileScanner
import asyncio
import sys
import re

cmd = "tcpdump --immediate-mode -l -g -k -n -i pktap,en0 ip or udp"


async def _read_stream(stream, cb):
    while True:
        line = await stream.readline()
        if line:
            cb(line.decode("utf-8").strip())
        else:
            break


async def _stream_subprocess(cmd, stdout_cb, stderr_cb):
    process = await asyncio.create_subprocess_exec(
        *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
    )

    await asyncio.wait(
        [
            _read_stream(process.stdout, stdout_cb),
            _read_stream(process.stderr, stderr_cb),
        ]
    )
    return await process.wait()


def execute(cmd, stdout_cb, stderr_cb):
    loop = asyncio.get_event_loop()
    rc = loop.run_until_complete(
        _stream_subprocess(
            cmd,
            stdout_cb,
            stderr_cb,
        )
    )
    loop.close()
    return rc


tcpdump_samples = """
21:52:33.105319 (en0, proc Google Chrome He:905, eproc Google Chrome He:905, svc BE, in, so) IP 192.168.1.200.8009 > 192.168.1.82.49983: Flags [.], ack 117, win 2046, options [nop,nop,TS val 2088759813 ecr 625396934], length 0

00:17:52.389193 (en0, proc Google Chrome He:905, svc BE, out, so) IP 192.168.1.82.49880 > 192.168.1.74.8009: Flags [P.], seq 111017849:111017959, ack 1418564688, win 2048, options [nop,nop,TS val 634067309 ecr 147440176], length 110

04:30:22.045424 (en0, proc rapportd:429, svc BE, out, so) IP 192.168.1.82.3722 > 192.168.1.90.3722: UDP, length 4

04:32:14.852119 (en0, proc mDNSResponder:176, svc BE, in, so) IP 192.168.1.74.5353 > 224.0.0.251.5353: 0 PTR (QM)? _googlecast._tcp.local. (40)

04:37:07.641576 (en0, proc mDNSResponder:176, eproc AppleConnectAgen:558, svc BE, out, so) IP6 2600:1700:43b0:d640:bd9b:525:de44:20d2.59694 > 2600:1700:43b0:d640::1.53: 58605+ Type65? wdg2.apple.com. (32)
"""


def parseTcpdumpLine(line):
    r = re.compile(
        """
    ^                       # start of line
    (?P<ts>[\d\.:]+)\s      # timestamp
    \((?P<md>.*?)\)\s       # PCAP-NG metadata
    (?P<fam>IP|IP6)\s       # Protocol family
    (?P<source>\S+)\s       # source IP / port
    > \s
    (?P<dst>\S+?):\s        # dst IP / port
    (                       # outside alternating group
      (                     # inside alt group group: TCP flags, sequence #, ack #,
                            #   window size, TCP options
        (Flags \s \[ (?P<flags>\S+) \],\s)?   # optional
        (seq\s(?P<seq>[\d:]+),\s)?            # optional
        (ack\s(?P<ack>\d+),\s)?               # optional
        (win\s(?P<win>\d+),\s)?               # optional
        (options\s\[(?P<options>.*?)\],\s)?   # optional
      ) | UDP, \s )                      # second inside alt group is for UDP packets
    (
      (length\s(?P<len>\d+)$)?           # optional length
      | 0 PTR \w*? (?P<dns>_\w+)         # second outside alt group is for DNS packets
    .*$)                                 # end
    """,
        re.X,
    )
    m = re.search(r, line)
    md = m.groupdict()
    # pp(md)

    od = {}
    try:
        if "ts" in md:
            od["ts"] = md["ts"]
        if "md" in md:
            a = list(map(lambda x: x.strip(), md["md"].split(",")))
            # print(f"a: {a}")
            od["ifx"] = a.pop(0)
            if "proc" in a[0]:
                od["proc"] = a.pop(0).lstrip("proc ")
            if "eproc" in a[0]:
                od["eproc"] = a.pop(0).lstrip("eproc ")
            if "svc" in a[0]:
                od["svc"] = a.pop(0).lstrip("svc ")
            od["dir"] = a.pop(0)
            # print(od)
        if "fam" in md:
            od["fam"] = md["fam"]
            # print(od)
        if "source" in md:
            s = md.get("source")
            # print(f"raw source type: {type(s)}, val: {s}")
            if od["fam"] == "IP":
                # print("fam IP")
                a = s.split(".")
                # print(a)
                od["sourceIP"] = ".".join(a[0:4])
                # print(od["sourceIP"])
                if len(a) > 4:
                    od["sourcePort"] = a[4]
                    # print(od["sourcePort"])

            else:
                od["sourceIP"] = s.split(".")[0]
                # print(od["sourceIP"])
                od["sourcePort"] = s.split(".")[1]
                # print(od["sourcePort"])

            # print(od)
        if "dst" in md:
            a = md["dst"].split(".")
            od["dstIP"] = ".".join(a[0:4])
            if len(a) > 4:
                od["dstPort"] = a[4]
            # print(od)
        if "length" in md:
            od["length"] = md["length"]
        si = od.get("sourceIP")
        di = od.get("dstIP")
        sp = od.get("sourcePort")
        dp = od.get("dstPort")
        col = set()
        col.add(si)
        col.add(di)
        if sp:
            col.add(sp)
        if dp:
            col.add(dp)

        c = ColorHash("".join(col))
        # print(f"color: {c.hex}")
        print(
            f"{od.get('ts')}|{di}|/{dp}"
            + f"|200|{od.get('length')}|1|{c.hex}|{si}:{sp}|{od['dir']}|{od.get('proc')}"
        )
    except:
        print(f"uh oh! {sys.exc_info()[0]}")
        sys.exit(1)

    # Example of standard logstalgia input format
    # 1571270436|192.168.10.5|/TCPDUMP/49418:example.com:https|200|1024
    # Example of extended logstalgia input format
    # 1371769989|127.0.0.1|/index.html|200|1024|1|00FF00|http://www.example.com/|Mozilla/5.0|webserver|1234
    # Extended field spec
    # timestamp | hostname| path | response code | response size | success |
    # response colour | referrer url | user agent | virtual host | pid/other


if __name__ == "__main__":
    print(
        execute(
            cmd.split(),
            lambda x: parseTcpdumpLine(x),
            lambda x: print("STDERR: %s" % x),
        )
    )

❯ sudo python -u lscap.py | logstalgia --hide-response-code --pitch-speed .5 --glow-intensity 0.2 -x