Skip to content

Instantly share code, notes, and snippets.

@maxenglander
Last active March 14, 2022 20:37
Show Gist options
  • Save maxenglander/157920844ab65e54e32f10597dbd14af to your computer and use it in GitHub Desktop.
Save maxenglander/157920844ab65e54e32f10597dbd14af to your computer and use it in GitHub Desktop.
ebpf_exporter configuration with a program to measure audit_log_start latency
programs:
- name: audit-log-start-latency
metrics:
histograms:
- name: function_latency_seconds
help: Latency of Linux kernel function
table: dist
bucket_type: fixed
bucket_keys:
- 1000 # 1ms
- 10000 # 10ms
- 100000 # 100ms
- 1000000 # 1s
- 10000000 # 10s
bucket_multiplier: 1e-06 # microseconds to seconds
labels:
- name: function
size: 8
decoders:
- name: ksym
- name: bucket
size: 8
decoders:
- name: uint
kprobes:
audit_log_start: trace_func_entry
kretprobes:
audit_log_start: trace_func_return
code: |
#include <uapi/linux/ptrace.h>
// Set up user-defined buckets.
const static u64 buckets[] = {
1000, // 1ms
10000, // 10ms
100000, // 100ms
1000000, // 1s
10000000 // 10s
};
const static int NUM_BUCKETS = sizeof(buckets) / sizeof(buckets[0]);
// Define the key used in the latency histogram.
typedef struct hist_key {
u64 ip;
u64 bucket;
} hist_key_t;
// Used to keep track of the start time of kernel routines.
BPF_HASH(start, u32);
// Used to keep track of the address of the kernel routine kprobe.
BPF_HASH(ipaddr, u32);
// Used to record the latency of kernel routines.
BPF_HISTOGRAM(dist, hist_key_t);
static u64 get_bucket(u64 value) {
for (int i = 0; i < NUM_BUCKETS; i++) {
if (value <= buckets[i]) {
return buckets[i];
}
}
// Cap at maximum value
return buckets[NUM_BUCKETS - 1];
}
// Called when a kprobe is entered.
int trace_func_entry(struct pt_regs *ctx)
{
// Get the process ID that resulted in this kprobe.
u64 pid_tgid = bpf_get_current_pid_tgid();
u32 pid = pid_tgid;
// Get and record the current kernel time (in nanoseconds).
u64 ts = bpf_ktime_get_ns();
start.update(&pid, &ts);
// Get and record the kprobe address.
u64 ip = PT_REGS_IP(ctx);
ipaddr.update(&pid, &ip);
return 0;
}
// Called when a kprobe returns.
int trace_func_return(struct pt_regs *ctx)
{
// Get the process ID that resulted in this kprobe.
u64 pid_tgid = bpf_get_current_pid_tgid();
u32 pid = pid_tgid;
// Will be used to retrieve start time and calculate latency.
u64 *tsp, delta;
// Retrieve the start time that was stored in trace_func_entry.
tsp = start.lookup(&pid);
// Return is no start time was found.
if (tsp == 0) {
return 0;
}
// Calculate the latency of the .
delta = bpf_ktime_get_ns() - *tsp;
// Convert nanoseconds to microseconds.
delta /= 1000;
// Remove the start time from the hash.
start.delete(&pid);
// Retrieve the kprobe address.
u64 ip, *ipp = ipaddr.lookup(&pid);
// Return if kprobe address not found.
if (ipp == 0) {
return 0;
}
ip = *ipp;
// Construct histogram key.
hist_key_t key;
// From ebpf_exporter docs:
// > Note that sometimes you can observe PT_REGS_IP being off by one.
// > You can subtract 1 in your code to make it point to the right
// > instruction that can be found /proc/kallsyms.
key.ip = ip - 1;
// Get the user-defined bucket of the latency, and increment the
// slot for that value in the latency histogram.
key.bucket = get_bucket(delta);
dist.increment(key);
// Increment the optional sum key.
hist_key_t max_key;
max_key.bucket = buckets[NUM_BUCKETS - 1] + 1;
max_key.ip = key.ip;
dist.increment(max_key, delta);
// Delete the kprobe address from the hash.
ipaddr.delete(&pid);
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment