fcntl_setlk_bcc_trace

debug_fcntl_setlk.c

#include <uapi/linux/ptrace.h>
#include <linux/sched.h>
#include <linux/fcntl.h>
#include <linux/fs.h>
#include <linux/path.h>
#include <linux/dcache.h>
#include <linux/limits.h>

#define ETYPE_SETLK                 1
#define ETYPE_CLOSE                 2
#define ETYPE_FILP_CLOSE            3
#define ETYPE_LOCKS_REMOVE_POSIX    4
#define ETYPE_DENTRY_OPEN           5

struct data_t {
	u32 etype;
	u32 pid;
	char comm[TASK_COMM_LEN];
	u32 result;
	u32 fd;
	u32 l_type;
	void* filp;
	char name[NAME_MAX];
};

BPF_RINGBUF_OUTPUT(events, 32);

static int filter_accept(struct data_t *data) {
	COMM_FILTER;

	return 1;
}

static struct data_t *init_data() {
	struct data_t *data;

	data = events.ringbuf_reserve(sizeof(struct data_t));
	if (!data) return NULL; // Failed to reserve space

	data->pid = bpf_get_current_pid_tgid();
	bpf_get_current_comm(&data->comm, sizeof(data->comm));
	if (!filter_accept(data)) {
		events.ringbuf_discard(data, 0 /* flags */);
		return NULL;
	}

	data->etype = 0;
	data->result = 0x55555555;
	data->fd = -1;
	data->l_type = -1;
	data->filp = NULL;
	data->name[0] = 0;

	return data;
}

static void set_simple_filename_from_filp(struct data_t *data, struct file *filp) {
	data->filp = filp;
	bpf_probe_read_kernel_str(data->name, sizeof(data->name), filp->f_path.dentry->d_name.name);
}

static void set_filename_from_filp(struct data_t *data, struct file *filp) {
	data->filp = filp;
	/* long bpf_d_path(struct path *path, char *buf, u32 sz) */
	if (bpf_d_path(&filp->f_path, data->name, sizeof(data->name)) < 0) {
		bpf_probe_read_kernel_str(data->name, sizeof(data->name), filp->f_path.dentry->d_name.name);
	}
}

KRETFUNC_PROBE(
	fcntl_setlk,
	unsigned int fd,
	struct file * filp,
	unsigned int cmd,
	struct flock * flock,
	int ret
) {
	struct data_t *data;

	if (cmd != F_SETLK && cmd != F_SETLKW) {
		return 0;
	}

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_SETLK;
	data->result = ret;
	data->fd = fd;
	data->l_type = flock->l_type;
	set_simple_filename_from_filp(data, filp);

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}

KFUNC_PROBE(
	close_fd,
	unsigned int fd
) {
	struct data_t *data;

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_CLOSE;
	data->fd = fd;

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}

KFUNC_PROBE(
	filp_close,
	struct file *filp,
	fl_owner_t id
) {
	struct data_t *data;

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_FILP_CLOSE;
	set_filename_from_filp(data, filp);

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}

KFUNC_PROBE(
	locks_remove_posix,
	struct file *filp,
	fl_owner_t owner
) {
	struct data_t *data;

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_LOCKS_REMOVE_POSIX;
	set_simple_filename_from_filp(data, filp);

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}


// dentry_open doesn't actually trigger...
#if 0
KRETFUNC_PROBE(
	dentry_open,
	const struct path *path,
	int flags,
	const struct cred *cred,
	struct file *result_filp
) {
	struct data_t *data;

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_DENTRY_OPEN;
	set_filename_from_filp(data, result_filp);

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}
#endif

KFUNC_PROBE(
	security_file_open,
	struct file *filp
) {
	struct data_t *data;

	data = init_data();
	if (!data) return 0; // no memory / filtered

	data->etype = ETYPE_DENTRY_OPEN;
	set_filename_from_filp(data, filp);

	events.ringbuf_submit(data, 0 /* flags */);

	return 0;
}

debug_fcntl_setlk.py

#!/usr/bin/env python3

from __future__ import annotations

import argparse
import dataclasses
import enum
import os
import os.path
import typing
import ctypes

import bcc


def _read_c_source() -> str:
    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'debug_fcntl_setlk.c')) as f:
        return f.read()


class EventType(enum.IntEnum):
    SETLK = 1
    CLOSE = 2
    FILP_CLOSE = 3
    LOCKS_REMOVE_POSIX = 4
    DENTRY_OPEN = 5


class EventData(typing.Protocol):
    etype: int  # EventType
    pid: int
    comm: bytes
    result: int
    fd: int
    l_type: int
    filp: int
    name: bytes


LockType = typing.Literal["shared", "exclusive"]|None


@dataclasses.dataclass(slots=True)
class File:
    ptr: int
    lock: LockType = None
    path: str = ""
    name: str = ""

    @property
    def filename(self) -> str:
        return self.path or self.name or '<unknown>'

    def seen_as_fd(self, pid: int, fd: int) -> None:
        if fd != -1 and not self.path:
            syml_name = f"/proc/{pid}/fd/{fd}"
            try:
                self.path = os.readlink(syml_name)
                #if self.path != self.name:
                #    print(f"[{pid}] fd {fd} is {self.path!r}")
            except OSError as e:
                #print(f"Failed to readlink [{fd} -> {self.name}]: {e}")
                pass


@dataclasses.dataclass(slots=True)
class Process:
    pid: int
    comm: str

    # map `struct file*` pointer to file
    files: dict[int, File] = dataclasses.field(default_factory=dict)
    # map fd to File (if known)
    open_fds: dict[int, File] = dataclasses.field(default_factory=dict)
    # orders of locks seen
    orders: set[tuple[str, ...]] = dataclasses.field(default_factory=set)

    def debug_locks(self, msg: str) -> None:
        if False:
            print(f"[{self.pid}] comm={self.comm} {msg}")

    def log(self, msg: str) -> None:
        print(f"[{self.pid}] comm={self.comm} {msg}")

    def trace_order(self) -> None:
        current_locks = tuple(
            file.filename
            for file in self.files.values()
            if file.lock
        )
        if current_locks in self.orders:
            return
        self.orders.add(current_locks)
        print(f"[{self.pid}] comm={self.comm}: New lock order observed: {current_locks}")

    def setlk(self, fd: int, filp: int, l_type: int, name: bytes, result: int) -> None:
        lock: LockType = None  # default / F_UNLCK == 2
        if l_type == 0: # F_RDLCK
            lock = "shared"
        elif l_type == 1:  # F_WRLCK
            lock = "exclusive"
        file = self.open_fds.get(fd, None)
        if file is None:
            file = self.files.get(filp, None)
        if file is None:
            self.files[filp] = file = File(ptr=filp, name=name.decode(errors='replace'))
            if fd != -1:
                self.open_fds[fd] = file
        file.seen_as_fd(self.pid, fd)
        if result == 0:
            if lock != file.lock:
                file.lock = lock
                if lock:
                    self.debug_locks(f"Locked {lock}: fd={fd}, path={file.filename}")
                    self.trace_order()
                else:
                    self.debug_locks(f"Unlock: fd={fd}, path={file.filename}")
        else:
            self.debug_locks(f"Locked {lock}: fd={fd}, path={file.filename}")

    def close(self, fd: int) -> None:
        file = self.open_fds.pop(fd, None)
        if not file is None:
            if file.lock:
                self.debug_locks(f"Unlock by close: fd={fd}, path={file.filename}")
                file.lock = None

    def close_filp(self, filp: int) -> None:
        file = self.files.pop(filp, None)
        if file is None:
            return
        if file.lock:
            self.debug_locks(f"Unlock by close: path={file.filename}")
            file.lock = None

    def locks_remove_posix(self, filp: int) -> None:
        file = self.files.get(filp, None)
        if file is None:
            return
        if file.lock:
            self.debug_locks(f"Unlock by locks_remove_posix: path={file.filename}")
            file.lock = None

    def dentry_open(self, filp: int, path: str) -> None:
        file = self.files.get(filp, None)
        if not file:
            self.files[filp] = file = File(ptr=filp, path=path)
        else:
            file.path = path
        # self.log(f"File opened: {file.filename}")


@dataclasses.dataclass(slots=True)
class Processes:
    processes: dict[int, Process] = dataclasses.field(default_factory=dict)

    def get_process(self, pid: int, comm: bytes) -> Process:
        proc = self.processes.get(pid)
        if proc is None:
            self.processes[pid] = proc = Process(pid=pid, comm=comm.decode(errors='replace'))
        return proc


def parse_args():
    parser = argparse.ArgumentParser(description="Debug fcntl(F_SETLK[W]) locks")
    parser.add_argument('comm', action='store', nargs='?')
    return parser.parse_args()


def main():
    args = parse_args()

    processes = Processes()

    source = _read_c_source()

    comm_filter = ""
    if args.comm:
        comm_bytes: bytes = args.comm.encode()
        comm_filter = ''.join([
            f"\tif (data->comm[{ndx}] != {chr(ch)!r}) return 0;\n"
            for ndx, ch in enumerate(comm_bytes + b'\0')
        ])

    source = source.replace('COMM_FILTER;', comm_filter)

    bpf = bcc.BPF(text=source)
    bpf_events: bcc.RingBuf = bpf["events"]

    def handle_event(ctx: typing.Any, data_ptr: ctypes._Pointer, data_size: int) -> typing.Literal[0]:
        data: EventData = bpf_events.event(data_ptr)
        # print(dict(etype=EventType(data.etype).name, pid=data.pid, comm=data.comm))
        process = processes.get_process(data.pid, data.comm)
        if data.etype == EventType.SETLK:
            process.setlk(data.fd, data.filp, data.l_type, data.name, data.result)
        elif data.etype == EventType.CLOSE:
            process.close(data.fd)
        elif data.etype == EventType.FILP_CLOSE:
            process.close_filp(data.filp)
        elif data.etype == EventType.LOCKS_REMOVE_POSIX:
            process.locks_remove_posix(data.filp)
        elif data.etype == EventType.DENTRY_OPEN:
            process.dentry_open(data.filp, data.name.decode(errors='replace'))
        return 0

    bpf_events.open_ring_buffer(handle_event)

    print("Waiting for events")
    while True:
        bpf.ring_buffer_poll(30)


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        pass