Skip to content

Instantly share code, notes, and snippets.

@rrampage

rrampage/sockpuppet.c

Last active October 5, 2026 16:09
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rrampage/92f0eb6bf56d7bb403aff069cc8f1d6b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rrampage/92f0eb6bf56d7bb403aff069cc8f1d6b to your computer and use it in GitHub Desktop.
Download ZIP
A userspace sandbox which uses SOCKS proxy to restrict network access (inspired by oniux)
Raw
sockpuppet.c
#define _GNU_SOURCE
/*
* sockpuppet.c - single-file Linux sandbox + userspace network broker
*
* Quick build:
* gcc -O2 -g -Wall -Wextra -Wformat -Wformat=2 -Wconversion \
* -Wimplicit-fallthrough -Werror=format-security \
* -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3 -D_GLIBCXX_ASSERTIONS \
* -fstack-clash-protection -fstack-protector-strong \
* -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer \
* -Wl,-z,relro -Wl,-z,now -Wl,--as-needed \
* -Wl,--no-copy-dt-needed-entries sockpuppet.c -o sockpuppet
*
* Common usage:
* ./sockpuppet /bin/sh
* Run a command with non-interactive stdio: stdin from /dev/null and
* stdout/stderr relayed through the parent.
*
* ./sockpuppet --interactive /bin/sh
* Run with a private PTY for shells, REPLs, and full-screen terminal apps.
*
* ./sockpuppet --allow-host=127.0.0.1:8080/tcp curl http://10.0.1.1:8080
* Allow the sandbox to reach a host-local service through the 10.0.1.x
* gateway mapping.
*
* ./sockpuppet --allow-direct=169.254.169.254/32:80/tcp curl ...
* Permit a direct-mode connection to a normally blocked special-use range.
*
* ./sockpuppet --publish=127.0.0.2:8080:8080/tcp python3 -m http.server 8080
* Publish a sandbox TCP service on an explicit host loopback alias. UDP
* uses the same grammar with /udp. 127.0.0.1 is intentionally rejected.
*
* ./sockpuppet --egress=none --publish=127.0.0.2:8080:8080/tcp ...
* Block child-initiated TCP/UDP egress while still permitting replies for
* active published inbound flows.
*
* ./sockpuppet --socks socks5://127.0.0.1:1080 curl https://example.com
* Route outbound traffic through a SOCKS5 proxy.
*
* To run under delegated cgroup containment from an external wrapper:
* systemd-run --user --scope --quiet --same-dir --collect \
* --property=Delegate=yes --property=MemoryMax=... \
* --property=MemoryHigh=... --property=TasksMax=... \
* --property=CPUQuota=... -- /path/to/sockpuppet ...
* Keep launch policy in the wrapper; this binary only checks whether it is
* already inside a delegated subtree.
*/
#include <arpa/inet.h>
/* Assertions are always on (STYLEGUIDE.md): a broken invariant crashes the
* broker, which tears the sandbox down, instead of continuing. */
#undef NDEBUG
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <pwd.h>
#include <linux/audit.h>
#include <linux/capability.h>
#include <linux/filter.h>
#include <linux/if.h>
#include <linux/if_link.h>
#include <linux/if_tun.h>
#include <linux/mount.h>
#include <linux/netlink.h>
#include <linux/openat2.h>
#include <linux/rtnetlink.h>
#include <linux/sched.h>
#include <linux/securebits.h>
#include <linux/seccomp.h>
#include <linux/virtio_net.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <stddef.h>
#include <poll.h>
#include <sched.h>
#include <signal.h>
#include <limits.h>
#include <stdio.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include <sys/random.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/un.h>
#include <sys/uio.h>
#include <sys/xattr.h>
#include <termios.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
/* Native x86-64, arm64 and riscv64 share this number (Linux 6.15).
* Older build headers must still deny it on newer running kernels. */
#if !defined(__NR_open_tree_attr) && \
(defined(__x86_64__) || defined(__aarch64__) || \
(defined(__riscv) && __riscv_xlen == 64))
#define __NR_open_tree_attr 467
#endif
#ifndef SOL_UDP
#define SOL_UDP IPPROTO_UDP
#endif
#ifndef UDP_GRO
#define UDP_GRO 104
#endif
#ifndef SOCKPUPPET_TCP_CORE_ONLY
typedef ssize_t (*sp_random_source_fn)(void *, size_t, unsigned int);
typedef int (*sp_recvmmsg_fn)(int, struct mmsghdr *, unsigned int, int,
struct timespec *);
typedef int (*sp_sendmmsg_fn)(int, struct mmsghdr *, unsigned int, int);
/* Steady-state authority minting must never wait for entropy. The indirection
* is intentionally narrow: focused tests replace only this syscall boundary
* to prove EAGAIN/short-read handling without depending on host entropy state. */
static sp_random_source_fn g_sp_random_source = getrandom;
static int sp_system_recvmmsg(int fd, struct mmsghdr *messages,
unsigned int count, int flags,
struct timespec *timeout) {
#ifdef __GLIBC__
return recvmmsg(fd, messages, count, flags, timeout);
#else
return recvmmsg(fd, messages, count, (unsigned int)flags, timeout);
#endif
}
static sp_recvmmsg_fn g_sp_recvmmsg = sp_system_recvmmsg;
static int sp_system_sendmmsg(int fd, struct mmsghdr *messages,
unsigned int count, int flags) {
#ifdef __GLIBC__
return sendmmsg(fd, messages, count, flags);
#else
return sendmmsg(fd, messages, count, (unsigned int)flags);
#endif
}
static sp_sendmmsg_fn g_sp_sendmmsg = sp_system_sendmmsg;
static int sp_random_exact_nonblocking(void *out, size_t len) {
ssize_t n;
if (len > (size_t)SSIZE_MAX) {
errno = EOVERFLOW;
return -1;
}
n = g_sp_random_source(out, len, GRND_NONBLOCK);
if (n == (ssize_t)len)
return 0;
if (n >= 0)
errno = EIO;
return -1;
}
#endif
#ifdef SOCKPUPPET_TCP_CORE_ONLY
/*
* The focused compile contract includes this translation-unit prefix and
* stops immediately after the typed TCP core. Poison authority-bearing APIs
* after the system headers have declared them: any later core reference is a
* compile error, while the ordinary single-file build remains unchanged.
*/
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC poison accept accept4 bind clock_gettime close connect epoll_ctl
#pragma GCC poison epoll_wait epoll_pwait epoll_pwait2 fcntl getrandom ioctl
#pragma GCC poison listen open openat pidfd_open poll ppoll read readv recv
#pragma GCC poison recvfrom recvmsg recvmmsg select send sendmmsg sendmsg sendto
#pragma GCC poison shutdown socket syscall time timerfd_create timerfd_settime
#pragma GCC poison write writev
#pragma GCC poison sp_counter_add sp_counter_inc
#else
#error "SOCKPUPPET_TCP_CORE_ONLY requires compiler identifier poisoning"
#endif
#endif
enum {
SP_ACCEPTED_PROFILE_VERSION = 1,
SP_TCP_CORE_MIN_MSS = 48,
SP_TCP_CORE_DEFAULT_MSS = 536,
SP_TCP_CORE_MAX_MSS = 1460,
SP_TCP_CORE_RCV_WSCALE = 3,
SP_TCP_CORE_PENDING_CAPACITY = 262144,
SP_TCP_CORE_MAX_EFFECTS = 8,
SP_TCP_CORE_MAX_SEGMENTS = 32,
SP_TCP_CORE_MAX_RTO_RETRIES = 8,
SP_TCP_CORE_K_RX = 6,
};
#define SP_TCP_CORE_RCV_QUANTUM \
(UINT64_C(1) << SP_TCP_CORE_RCV_WSCALE)
#define SP_TCP_CORE_ADVERTISABLE_CAPACITY \
(((uint64_t)SP_TCP_CORE_PENDING_CAPACITY / SP_TCP_CORE_RCV_QUANTUM) * \
SP_TCP_CORE_RCV_QUANTUM)
#define SP_TCP_CORE_UNADVERTISED_TAIL \
((uint64_t)SP_TCP_CORE_PENDING_CAPACITY - \
SP_TCP_CORE_ADVERTISABLE_CAPACITY)
_Static_assert(SP_TCP_CORE_RCV_WSCALE <= 14,
"receive-window shift exceeds RFC 7323 maximum");
_Static_assert(SP_TCP_CORE_ADVERTISABLE_CAPACITY >= SP_TCP_CORE_RCV_QUANTUM,
"receive window must contain a complete scale quantum");
_Static_assert(SP_TCP_CORE_ADVERTISABLE_CAPACITY / SP_TCP_CORE_RCV_QUANTUM <=
UINT16_MAX,
"scaled receive window does not fit the wire field");
_Static_assert(SP_TCP_CORE_UNADVERTISED_TAIL < SP_TCP_CORE_RCV_QUANTUM,
"receive-window floor may hide at most one partial quantum");
enum sp_effect_type {
SP_FX_OPEN_STREAM = 1,
SP_FX_TCP_EMIT_BATCH,
SP_FX_TCP_EMIT_CONTROL,
SP_FX_STREAM_SEND,
SP_FX_STREAM_CONSUME,
SP_FX_INGRESS_DISPOSE,
SP_FX_STREAM_SHUTDOWN,
SP_FX_CLOSE_FLOW,
};
typedef struct { uint64_t value; } sp_flow_id;
typedef struct { uint64_t value; } sp_generation_id;
typedef struct { uint64_t value; } sp_effect_id;
typedef struct { uint64_t value; } sp_ingress_id;
typedef struct { uint64_t value; } sp_capability_id;
typedef struct { uint64_t value; } sp_time_ns;
typedef struct { uint64_t value; } sp_stream_offset;
typedef struct { uint64_t value; } sp_byte_count;
typedef struct { uint64_t value; } sp_core_sequence;
typedef struct { uint64_t value; } sp_segment_count;
struct sp_flow_key {
sp_flow_id flow;
sp_generation_id generation;
};
enum sp_construct_result {
SP_CONSTRUCT_OK = 0,
SP_CONSTRUCT_INTERNAL_INVARIANT = 3,
};
/* Named annotations keep intentional test-build, oracle, and platform helper
* reachability reviewable. `SP_TEST_API` means a helper is deliberately
* visible to focused single-file tests and may be unused in such a translation
* unit; it does not mean that the shipping program cannot call it. */
#define SP_TEST_API __attribute__((unused))
#define SP_ORACLE_API __attribute__((unused))
static inline struct sp_flow_key sp_flow_key_make(sp_flow_id flow,
sp_generation_id generation) {
return (struct sp_flow_key){.flow = flow, .generation = generation};
}
enum sp_tcp_policy_role {
SP_TCP_ROLE_DIRECT = 1,
SP_TCP_ROLE_SOCKS,
SP_TCP_ROLE_PUBLISH,
};
enum sp_tcp_open_role {
SP_TCP_OPEN_PASSIVE = 1,
SP_TCP_OPEN_ACTIVE,
};
enum sp_tcp_core_state {
SP_TCP_CORE_CLOSED = 0,
SP_TCP_CORE_CONNECTING_PASSIVE,
SP_TCP_CORE_CONNECTING_ACTIVE,
SP_TCP_CORE_SYN_RECEIVED,
SP_TCP_CORE_SYN_SENT,
SP_TCP_CORE_ESTABLISHED,
SP_TCP_CORE_FIN_WAIT_1,
SP_TCP_CORE_FIN_WAIT_2,
SP_TCP_CORE_CLOSE_WAIT,
SP_TCP_CORE_CLOSING,
SP_TCP_CORE_LAST_ACK,
SP_TCP_CORE_TIME_WAIT,
};
/* Independent deadlines. Ordering within this enum is not the firing
* precedence; sp_tcp_timer_priority() defines that ladder. */
enum sp_tcp_timer_kind {
SP_TCP_TIMER_HANDSHAKE = 1,
SP_TCP_TIMER_RTO,
SP_TCP_TIMER_PERSIST,
SP_TCP_TIMER_SEND_AGE,
SP_TCP_TIMER_RECEIVE_AGE,
SP_TCP_TIMER_CLOSE,
SP_TCP_TIMER_TIME_WAIT,
};
enum sp_tcp_core_result {
SP_TCP_CORE_OK = 0,
SP_TCP_CORE_REJECTED,
SP_TCP_CORE_BAD_EVENT,
SP_TCP_CORE_EFFECT_OVERFLOW,
};
enum sp_tcp_core_flag {
SP_TCP_CORE_FIN = 0x01,
SP_TCP_CORE_SYN = 0x02,
SP_TCP_CORE_RST = 0x04,
SP_TCP_CORE_PSH = 0x08,
SP_TCP_CORE_ACK = 0x10,
};
struct sp_tcp_tuple {
uint32_t local_addr;
uint32_t peer_addr;
uint16_t local_port;
uint16_t peer_port;
};
struct sp_tcp_peer_options {
uint16_t mss;
uint8_t mss_present;
uint8_t ws_present;
uint8_t ws_raw;
};
/* Products remain deliberately distinct even when their current payloads have
* the same shape. Exact product identity makes cross-family substitution
* mechanically visible to the compiler. */
struct sp_typed_event_context {
struct sp_flow_key key;
sp_time_ns now;
sp_core_sequence control_sequence;
sp_core_sequence causal_fence;
};
struct sp_ev_flow_admitted {
struct sp_flow_key key;
sp_time_ns now;
struct sp_tcp_tuple tuple;
enum sp_tcp_policy_role policy_role;
enum sp_tcp_open_role open_role;
uint32_t local_iss;
uint32_t peer_isn;
uint16_t peer_window;
uint16_t peer_mss;
uint8_t needs_connect;
uint8_t local_rcv_wscale;
uint8_t passive_syn_present;
uint8_t peer_mss_present;
uint8_t peer_ws_present;
uint8_t peer_ws;
};
struct sp_ev_tcp_segment {
struct sp_typed_event_context context;
sp_ingress_id ingress;
sp_byte_count payload_length;
uint32_t seq;
uint32_t ack;
uint16_t window;
uint16_t mss;
uint8_t flags;
uint8_t mss_present;
uint8_t ws_present;
uint8_t ws;
};
enum sp_connect_outcome {
SP_CONNECT_SUCCEEDED = 1,
SP_CONNECT_FAILED,
};
struct sp_ev_connect_result {
struct sp_typed_event_context context;
sp_effect_id effect;
enum sp_connect_outcome outcome;
};
enum sp_send_outcome {
SP_SEND_PROGRESS = 1,
SP_SEND_RETRY,
SP_SEND_FAILED,
};
struct sp_ev_stream_send_result {
struct sp_typed_event_context context;
sp_effect_id effect;
sp_byte_count accepted;
sp_byte_count retained;
enum sp_send_outcome outcome;
};
struct sp_ev_stream_consumed {
struct sp_typed_event_context context;
sp_effect_id effect;
sp_byte_count accepted;
};
struct sp_ev_stream_eof {
struct sp_typed_event_context context;
sp_stream_offset stream_tail;
};
enum sp_emit_outcome {
SP_EMIT_ACCEPTED = 1,
SP_EMIT_RETAINED,
SP_EMIT_FAILED,
};
struct sp_ev_tun_emit_result {
struct sp_typed_event_context context;
sp_effect_id effect;
sp_byte_count accepted;
enum sp_emit_outcome outcome;
};
struct sp_ev_credit {
struct sp_typed_event_context context;
sp_stream_offset stream_tail;
sp_byte_count pending_capacity;
sp_segment_count tun_segments;
};
struct sp_ev_timer_fired {
struct sp_typed_event_context context;
enum sp_tcp_timer_kind kind;
sp_time_ns deadline;
sp_generation_id timer_generation;
};
struct sp_ev_ingress_disposed {
struct sp_typed_event_context context;
sp_effect_id effect;
};
struct sp_typed_effect_context {
struct sp_flow_key key;
sp_effect_id effect;
};
struct sp_fx_open_stream {
struct sp_typed_effect_context context;
};
struct sp_fx_tcp_emit_batch {
struct sp_typed_effect_context context;
sp_stream_offset start;
sp_stream_offset end;
sp_core_sequence seq;
sp_core_sequence ack;
uint32_t flags;
uint32_t attempt;
uint16_t window;
uint16_t mss;
uint8_t terminal;
};
struct sp_fx_tcp_emit_control {
struct sp_typed_effect_context context;
sp_core_sequence seq;
sp_core_sequence ack;
uint32_t flags;
uint32_t attempt;
uint16_t window;
uint16_t mss;
uint8_t ws_present;
uint8_t ws;
uint8_t terminal;
};
struct sp_fx_stream_send {
struct sp_typed_effect_context context;
sp_capability_id capability;
sp_stream_offset start;
sp_stream_offset end;
};
struct sp_fx_stream_consume {
struct sp_typed_effect_context context;
sp_stream_offset start;
sp_stream_offset end;
};
struct sp_fx_ingress_dispose {
struct sp_typed_effect_context context;
sp_ingress_id ingress;
};
struct sp_fx_stream_shutdown {
struct sp_typed_effect_context context;
};
struct sp_fx_close_flow {
struct sp_typed_effect_context context;
};
struct sp_typed_effect {
enum sp_effect_type type;
union {
struct sp_fx_open_stream open_stream;
struct sp_fx_tcp_emit_batch tcp_emit_batch;
struct sp_fx_tcp_emit_control tcp_emit_control;
struct sp_fx_stream_send stream_send;
struct sp_fx_stream_consume stream_consume;
struct sp_fx_ingress_dispose ingress_dispose;
struct sp_fx_stream_shutdown stream_shutdown;
struct sp_fx_close_flow close_flow;
} value;
};
struct sp_typed_effect_list {
struct sp_typed_effect items[SP_TCP_CORE_MAX_EFFECTS];
size_t count;
};
static const struct sp_typed_effect_context *
sp_typed_effect_context(const struct sp_typed_effect *effect) {
switch (effect->type) {
case SP_FX_OPEN_STREAM:
return &effect->value.open_stream.context;
case SP_FX_TCP_EMIT_BATCH:
return &effect->value.tcp_emit_batch.context;
case SP_FX_TCP_EMIT_CONTROL:
return &effect->value.tcp_emit_control.context;
case SP_FX_STREAM_SEND:
return &effect->value.stream_send.context;
case SP_FX_STREAM_CONSUME:
return &effect->value.stream_consume.context;
case SP_FX_INGRESS_DISPOSE:
return &effect->value.ingress_dispose.context;
case SP_FX_STREAM_SHUTDOWN:
return &effect->value.stream_shutdown.context;
case SP_FX_CLOSE_FLOW:
return &effect->value.close_flow.context;
}
return NULL;
}
_Static_assert(sizeof(sp_flow_id) == sizeof(uint64_t),
"semantic IDs must remain zero-overhead wrappers");
_Static_assert(sizeof(struct sp_typed_event_context) <= 48,
"hot typed event context exceeded its reviewed ceiling");
_Static_assert(sizeof(struct sp_ev_tcp_segment) <= 96,
"hot scalar segment event exceeded its reviewed ceiling");
_Static_assert(sizeof(struct sp_fx_tcp_emit_batch) <= 96,
"hot TCP emit effect exceeded its reviewed ceiling");
struct sp_tcp_live_effect {
uint64_t id;
uint64_t start;
uint64_t end;
uint64_t ingress_id;
uint8_t retransmission;
};
struct sp_tcp_core {
enum sp_tcp_core_state state;
uint64_t flow;
uint64_t generation;
enum sp_tcp_policy_role policy_role;
enum sp_tcp_open_role open_role;
uint64_t next_effect_id;
uint64_t last_now_ns;
uint64_t applied_control_seq;
uint64_t iss_abs;
uint64_t irs_abs;
uint64_t data_seq0_abs;
uint64_t snd_una_abs;
uint64_t snd_nxt_high_abs;
uint64_t snd_wl1_abs;
uint64_t snd_wl2_abs;
uint64_t snd_wnd;
uint64_t stream_consumed;
uint64_t consume_target;
uint64_t stream_acked;
uint64_t stream_injected;
uint64_t stream_tail;
uint64_t final_stream_length;
uint8_t stream_eof;
uint8_t fin_injected;
uint8_t fin_acked;
uint64_t rcv_acked_abs;
uint64_t rx_pending_end_abs;
uint64_t rx_authorized_end_abs;
uint64_t rcv_adv_right_abs;
uint64_t pending_capacity;
uint8_t peer_fin_seen;
uint64_t peer_fin_abs;
uint16_t effective_send_mss;
uint8_t snd_wind_shift;
uint8_t rcv_wind_shift;
uint8_t snd_ws_ok;
uint64_t tun_segment_credit;
struct sp_tcp_live_effect connect_effect;
struct sp_tcp_live_effect send_caps[SP_TCP_CORE_K_RX];
unsigned send_cap_count;
struct sp_tcp_live_effect stream_consume_effect;
struct sp_tcp_live_effect tun_emit_effect;
struct sp_tcp_live_effect ingress_dispose_effect;
uint64_t handshake_deadline_ns;
uint64_t rto_deadline_ns;
uint64_t persist_deadline_ns;
uint64_t send_age_deadline_ns;
uint64_t receive_age_deadline_ns;
uint64_t close_deadline_ns;
uint64_t timewait_deadline_ns;
uint64_t timer_generation;
uint64_t timer_gen[SP_TCP_TIMER_TIME_WAIT + 1];
uint32_t persist_probes;
uint32_t rto_retries;
uint64_t rto_first_ns;
/* RTO recovery invariant: backoff history collapses only once new data
* beyond this recovery point is unambiguously acknowledged. */
uint64_t rto_recovery_point_abs;
/* Fast-retransmit invariant: one bounded episode per recovery point. */
uint64_t dup_ack_ref_abs;
uint32_t dup_ack_count;
uint64_t fast_retransmit_point_abs;
uint8_t fast_retransmit_used;
uint64_t rtt_sample_end_abs;
uint64_t rtt_sample_sent_ns;
uint64_t srtt_ns;
uint64_t rttvar_ns;
uint64_t rtt_samples;
uint8_t rtt_sample_ambiguous;
};
void sp_tcp_core_init(struct sp_tcp_core *core);
int sp_tcp_assert_invariants(const struct sp_tcp_core *core);
uint64_t sp_tcp_unwrap_u32(uint32_t wire, uint64_t checkpoint);
uint16_t sp_tcp_encode_window(const struct sp_tcp_core *core,
uint64_t desired_right,
uint64_t physical_capacity,
uint64_t *encoded_right);
uint64_t sp_tcp_next_deadline_ns(const struct sp_tcp_core *core,
enum sp_tcp_timer_kind *kind_out);
uint64_t sp_tcp_ns_until(uint64_t deadline_ns, uint64_t now_ns);
/* Frozen timer profile. The 200 ms minimum RTO is a deliberate
* nonconformance to RFC 6298's one-second minimum, and the 5 s maximum is a
* deliberate nonconformance to its 60-second minimum permitted maximum; both
* are accepted only for the private synthetic child link and are recorded in
* the deviation register. The one-second initial value is retained. */
enum {
SP_TCP_HANDSHAKE_NS = 10000000000ULL,
SP_TCP_RTO_NS = 1000000000ULL,
SP_TCP_RTO_MIN_NS = 200000000ULL,
SP_TCP_RTO_MAX_NS = 5000000000ULL,
SP_TCP_CLOSE_NS = 120000000000ULL,
SP_TCP_SEND_AGE_NS = 120000000000ULL,
SP_TCP_RECEIVE_AGE_NS = 120000000000ULL,
SP_TCP_TIME_WAIT_NS = 240000000000ULL,
SP_TCP_RTO_MAX_AGE_NS = 300000000000ULL,
SP_TCP_DUP_ACK_THRESHOLD = 3,
};
/* Persist probe schedule: 1, 2, 4, 8, 16, then 30 s capped. */
static const uint64_t sp_persist_schedule_ns[] = {
1000000000ULL, 2000000000ULL, 4000000000ULL,
8000000000ULL, 16000000000ULL, 30000000000ULL,
};
static uint64_t sp_min_u64(uint64_t a, uint64_t b) { return a < b ? a : b; }
static uint64_t sp_persist_delay_ns(uint32_t probes) {
size_t last = (sizeof(sp_persist_schedule_ns) /
sizeof(sp_persist_schedule_ns[0])) - 1;
return sp_persist_schedule_ns[probes < last ? probes : last];
}
/* Unmeasured flows use the one-second initial RTO; once RTT samples exist the
* base is the RFC 6298 estimator clamped into [200 ms, 5 s]. Backoff doubles
* the base and saturates at the same 5 s ceiling, so the eight permitted
* expiries of an unmeasured flow fall at 1, 2, 4, 5, 5, 5, 5, and 5 seconds
* and terminate the flow at 32 seconds absent ACK progress. */
static uint64_t sp_rto_base_ns(const struct sp_tcp_core *core) {
uint64_t base;
if (core->rtt_samples == 0)
return SP_TCP_RTO_NS;
base = core->srtt_ns + 4 * core->rttvar_ns;
if (base < SP_TCP_RTO_MIN_NS)
base = SP_TCP_RTO_MIN_NS;
return sp_min_u64(base, SP_TCP_RTO_MAX_NS);
}
static uint64_t sp_rto_delay_ns_from(uint64_t base, uint32_t retries) {
uint32_t shift = retries > 6 ? 6 : retries;
if (base >= SP_TCP_RTO_MAX_NS >> shift)
return SP_TCP_RTO_MAX_NS;
return sp_min_u64(base << shift, SP_TCP_RTO_MAX_NS);
}
static uint64_t sp_rto_delay_ns(const struct sp_tcp_core *core,
uint32_t retries) {
return sp_rto_delay_ns_from(sp_rto_base_ns(core), retries);
}
/* Timer firing precedence. When several deadlines for one flow are due at the
* same instant, terminal absolute deadlines run before RTO and persist so no
* extra packet is emitted after expiry:
*
* handshake/connect absolute deadline
* > send/receive/close resource deadline
* > RTO retry-limit terminal transition
* > retransmission or persist probe
* > TIME-WAIT expiry/other maintenance
*
* The RTO timer occupies one rank; whether a given expiry is the retry-limit
* terminal transition or an ordinary retransmission is decided in sp_timer(). */
static unsigned sp_tcp_timer_priority(enum sp_tcp_timer_kind kind) {
switch (kind) {
case SP_TCP_TIMER_HANDSHAKE:
return 1;
case SP_TCP_TIMER_SEND_AGE:
case SP_TCP_TIMER_RECEIVE_AGE:
case SP_TCP_TIMER_CLOSE:
return 2;
case SP_TCP_TIMER_RTO:
return 3;
case SP_TCP_TIMER_PERSIST:
return 4;
case SP_TCP_TIMER_TIME_WAIT:
return 5;
}
return 6;
}
static uint64_t *sp_timer_slot(struct sp_tcp_core *core,
enum sp_tcp_timer_kind kind) {
switch (kind) {
case SP_TCP_TIMER_HANDSHAKE:
return &core->handshake_deadline_ns;
case SP_TCP_TIMER_RTO:
return &core->rto_deadline_ns;
case SP_TCP_TIMER_PERSIST:
return &core->persist_deadline_ns;
case SP_TCP_TIMER_SEND_AGE:
return &core->send_age_deadline_ns;
case SP_TCP_TIMER_RECEIVE_AGE:
return &core->receive_age_deadline_ns;
case SP_TCP_TIMER_CLOSE:
return &core->close_deadline_ns;
case SP_TCP_TIMER_TIME_WAIT:
return &core->timewait_deadline_ns;
}
return NULL;
}
/* Arms or cancels one independent deadline. Every arm takes a fresh
* generation from the flow's monotonic counter, so a cancel/rearm that lands
* on the identical absolute deadline still invalidates the in-flight event. */
static void sp_arm_timer(struct sp_tcp_core *core, enum sp_tcp_timer_kind kind,
uint64_t deadline_ns) {
uint64_t *slot = sp_timer_slot(core, kind);
if (slot == NULL)
return;
*slot = deadline_ns;
core->timer_gen[kind] = deadline_ns != 0 ? ++core->timer_generation : 0;
}
/* True when a fired timer event still refers to the currently armed deadline.
* A zero event generation is accepted on deadline equality alone so producers
* that predate generation stamping stay compatible. */
static int sp_timer_current(const struct sp_tcp_core *core,
enum sp_tcp_timer_kind kind, uint64_t deadline_ns,
uint64_t generation) {
uint64_t armed = *sp_timer_slot((struct sp_tcp_core *)core, kind);
if (armed == 0 || deadline_ns != armed)
return 0;
return generation == 0 || generation == core->timer_gen[kind];
}
/* Resource deadlines. Each bounds how long one direction may hold an
* unfinished obligation. `progress` distinguishes a genuine advance of the
* acknowledged edge, which restarts the deadline, from merely acquiring new
* work, which only arms an idle timer. Without that distinction a sender
* injecting into a black hole would refresh its own deadline forever, and the
* deadline could move later for reasons this invariant forbids. */
static void sp_update_send_age(struct sp_tcp_core *core, uint64_t now_ns,
int progress) {
int outstanding = core->stream_acked < core->stream_injected ||
(core->fin_injected && !core->fin_acked);
if (!outstanding) {
sp_arm_timer(core, SP_TCP_TIMER_SEND_AGE, 0);
return;
}
if (progress || core->send_age_deadline_ns == 0)
sp_arm_timer(core, SP_TCP_TIMER_SEND_AGE, now_ns + SP_TCP_SEND_AGE_NS);
}
static void sp_update_receive_age(struct sp_tcp_core *core, uint64_t now_ns,
int progress) {
int outstanding = core->rx_authorized_end_abs > core->rcv_acked_abs;
if (!outstanding) {
sp_arm_timer(core, SP_TCP_TIMER_RECEIVE_AGE, 0);
return;
}
if (progress || core->receive_age_deadline_ns == 0)
sp_arm_timer(core, SP_TCP_TIMER_RECEIVE_AGE,
now_ns + SP_TCP_RECEIVE_AGE_NS);
}
/* Returns the flow's next deadline, or 0 when no timer is armed. Earliest
* deadline wins; equal deadlines break by the precedence ladder above. This is
* a pure query so the scheduler can recompute the epoll_pwait2 timeout from
* the minimum across flows without reading the clock. */
uint64_t sp_tcp_next_deadline_ns(const struct sp_tcp_core *core,
enum sp_tcp_timer_kind *kind_out) {
static const enum sp_tcp_timer_kind kinds[] = {
SP_TCP_TIMER_HANDSHAKE, SP_TCP_TIMER_SEND_AGE,
SP_TCP_TIMER_RECEIVE_AGE, SP_TCP_TIMER_CLOSE,
SP_TCP_TIMER_RTO, SP_TCP_TIMER_PERSIST,
SP_TCP_TIMER_TIME_WAIT,
};
const uint64_t deadlines[] = {
core->handshake_deadline_ns, core->send_age_deadline_ns,
core->receive_age_deadline_ns, core->close_deadline_ns,
core->rto_deadline_ns, core->persist_deadline_ns,
core->timewait_deadline_ns,
};
uint64_t best = 0;
unsigned best_priority = 0;
size_t i;
for (i = 0; i < sizeof(kinds) / sizeof(kinds[0]); ++i) {
unsigned priority;
if (deadlines[i] == 0)
continue;
priority = sp_tcp_timer_priority(kinds[i]);
if (best != 0 && (deadlines[i] > best ||
(deadlines[i] == best && priority >= best_priority)))
continue;
best = deadlines[i];
best_priority = priority;
if (kind_out != NULL)
*kind_out = kinds[i];
}
return best;
}
/* Nanoseconds remaining until an absolute monotonic deadline, saturating at
* zero. Callers pass the clock in so fake-clock tests can drive every
* boundary; the wait adapter converts the result to the epoll_pwait2 timespec. */
uint64_t sp_tcp_ns_until(uint64_t deadline_ns, uint64_t now_ns) {
if (deadline_ns == 0 || deadline_ns <= now_ns)
return 0;
return deadline_ns - now_ns;
}
uint64_t sp_tcp_unwrap_u32(uint32_t wire, uint64_t checkpoint) {
uint64_t candidate = (checkpoint & ~UINT64_C(0xffffffff)) | wire;
if (candidate + UINT64_C(0x80000000) < checkpoint)
candidate += UINT64_C(0x100000000);
else if (candidate > checkpoint + UINT64_C(0x80000000) &&
candidate >= UINT64_C(0x100000000))
candidate -= UINT64_C(0x100000000);
return candidate;
}
static uint32_t sp_tcp_wire(uint64_t absolute) {
return (uint32_t)absolute;
}
void sp_tcp_core_init(struct sp_tcp_core *core) {
memset(core, 0, sizeof(*core));
core->state = SP_TCP_CORE_CLOSED;
core->next_effect_id = 1;
core->pending_capacity = SP_TCP_CORE_PENDING_CAPACITY;
}
static int sp_effect_reserve(struct sp_tcp_core *core,
struct sp_typed_effect_list *list,
struct sp_typed_effect **effect) {
if (list->count >= SP_TCP_CORE_MAX_EFFECTS ||
core->next_effect_id == UINT64_MAX)
return -1;
*effect = &list->items[list->count++];
memset(*effect, 0, sizeof(**effect));
core->next_effect_id++;
return 0;
}
#define SP_DEFINE_EFFECT_ADD(name_, tag_, member_, product_) \
static int name_(struct sp_tcp_core *core, \
struct sp_typed_effect_list *list, product_ **out) { \
struct sp_typed_effect *item; \
struct sp_typed_effect_context context; \
if (sp_effect_reserve(core, list, &item) < 0) \
return -1; \
context.key = sp_flow_key_make((sp_flow_id){.value = core->flow}, \
(sp_generation_id){.value = \
core->generation}); \
context.effect.value = core->next_effect_id - 1; \
item->type = tag_; \
item->value.member_.context = context; \
*out = &item->value.member_; \
return 0; \
}
SP_DEFINE_EFFECT_ADD(sp_effect_add_open_stream, SP_FX_OPEN_STREAM, open_stream,
struct sp_fx_open_stream)
SP_DEFINE_EFFECT_ADD(sp_effect_add_tcp_emit_batch, SP_FX_TCP_EMIT_BATCH,
tcp_emit_batch, struct sp_fx_tcp_emit_batch)
SP_DEFINE_EFFECT_ADD(sp_effect_add_tcp_emit_control, SP_FX_TCP_EMIT_CONTROL,
tcp_emit_control, struct sp_fx_tcp_emit_control)
SP_DEFINE_EFFECT_ADD(sp_effect_add_stream_send, SP_FX_STREAM_SEND, stream_send,
struct sp_fx_stream_send)
SP_DEFINE_EFFECT_ADD(sp_effect_add_stream_consume, SP_FX_STREAM_CONSUME,
stream_consume, struct sp_fx_stream_consume)
SP_DEFINE_EFFECT_ADD(sp_effect_add_ingress_dispose, SP_FX_INGRESS_DISPOSE,
ingress_dispose, struct sp_fx_ingress_dispose)
SP_DEFINE_EFFECT_ADD(sp_effect_add_stream_shutdown, SP_FX_STREAM_SHUTDOWN,
stream_shutdown, struct sp_fx_stream_shutdown)
SP_DEFINE_EFFECT_ADD(sp_effect_add_close_flow, SP_FX_CLOSE_FLOW, close_flow,
struct sp_fx_close_flow)
#undef SP_DEFINE_EFFECT_ADD
uint16_t sp_tcp_encode_window(const struct sp_tcp_core *core,
uint64_t desired_right,
uint64_t physical_capacity,
uint64_t *encoded_right) {
uint64_t scale;
uint64_t advertisable_capacity;
uint64_t distance;
uint64_t field;
if (core == NULL || encoded_right == NULL || core->rcv_wind_shift > 14 ||
desired_right < core->rcv_acked_abs)
return 0;
scale = core->snd_ws_ok ? UINT64_C(1) << core->rcv_wind_shift : 1;
advertisable_capacity = (physical_capacity / scale) * scale;
distance = desired_right - core->rcv_acked_abs;
field = distance / scale + (distance % scale != 0);
if (field > UINT16_MAX || field * scale > advertisable_capacity)
return 0;
*encoded_right = core->rcv_acked_abs + field * scale;
return (uint16_t)field;
}
static uint16_t sp_current_window(struct sp_tcp_core *core) {
uint64_t scale = core->snd_ws_ok ? UINT64_C(1) << core->rcv_wind_shift : 1;
uint64_t advertisable_capacity =
(core->pending_capacity / scale) * scale;
uint64_t maximum;
uint64_t encoded_right = core->rcv_adv_right_abs;
uint16_t field;
if (UINT64_MAX - core->rcv_acked_abs < advertisable_capacity)
return 0;
maximum = core->rcv_acked_abs + advertisable_capacity;
if (maximum > core->rcv_adv_right_abs)
core->rcv_adv_right_abs = maximum;
field = sp_tcp_encode_window(core, core->rcv_adv_right_abs,
core->pending_capacity, &encoded_right);
if (field != 0 || core->rcv_adv_right_abs == core->rcv_acked_abs)
core->rcv_adv_right_abs = encoded_right;
return field;
}
static int sp_emit_control(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects, uint8_t flags,
uint64_t seq_abs, uint64_t ack_abs, int syn_options) {
struct sp_fx_tcp_emit_control *effect;
if (sp_effect_add_tcp_emit_control(core, effects, &effect) < 0)
return -1;
effect->flags = flags;
effect->seq.value = seq_abs;
effect->ack.value = ack_abs;
if ((flags & SP_TCP_CORE_SYN) != 0) {
uint64_t window = core->pending_capacity;
if (window > UINT16_MAX)
window = UINT16_MAX;
effect->window = (uint16_t)window;
} else {
effect->window = sp_current_window(core);
}
if (syn_options) {
effect->mss = SP_TCP_CORE_MAX_MSS;
effect->ws_present = core->open_role == SP_TCP_OPEN_ACTIVE ||
core->snd_ws_ok;
effect->ws = core->rcv_wind_shift;
}
return 0;
}
static int sp_emit_current_ack(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
return sp_emit_control(core, effects, SP_TCP_CORE_ACK,
core->snd_nxt_high_abs, core->rcv_acked_abs, 0);
}
static int sp_emit_ingress_dispose(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects,
uint64_t ingress_id) {
struct sp_fx_ingress_dispose *effect;
if (ingress_id == 0 || core->ingress_dispose_effect.id != 0)
return 0;
if (sp_effect_add_ingress_dispose(core, effects, &effect) < 0)
return -1;
effect->ingress.value = ingress_id;
core->ingress_dispose_effect.id = effect->context.effect.value;
core->ingress_dispose_effect.ingress_id = ingress_id;
return 0;
}
static int sp_peer_options_apply(struct sp_tcp_core *core,
const struct sp_tcp_peer_options *options,
int passive) {
uint16_t peer_mss;
if (!options->ws_present && options->ws_raw != 0)
return -1;
if (options->mss_present) {
if (options->mss < SP_TCP_CORE_MIN_MSS)
return -1;
peer_mss = options->mss;
} else {
if (options->mss != 0)
return -1;
peer_mss = SP_TCP_CORE_DEFAULT_MSS;
}
core->effective_send_mss =
peer_mss < SP_TCP_CORE_MAX_MSS ? peer_mss : SP_TCP_CORE_MAX_MSS;
if (options->ws_present) {
core->snd_wind_shift = options->ws_raw > 14 ? 14 : options->ws_raw;
core->snd_ws_ok = 1;
if (passive)
core->rcv_wind_shift = SP_TCP_CORE_RCV_WSCALE;
} else {
if (options->ws_raw != 0)
return -1;
core->snd_wind_shift = 0;
core->rcv_wind_shift = 0;
core->snd_ws_ok = 0;
}
return 0;
}
static int sp_admission_valid(const struct sp_ev_flow_admitted *admission) {
if (admission->needs_connect > 1 || admission->key.flow.value == 0 ||
admission->key.generation.value == 0 ||
admission->tuple.local_port == 0 || admission->tuple.peer_port == 0 ||
admission->local_rcv_wscale > 14 ||
admission->local_rcv_wscale != SP_TCP_CORE_RCV_WSCALE ||
admission->policy_role < SP_TCP_ROLE_DIRECT ||
admission->policy_role > SP_TCP_ROLE_PUBLISH ||
admission->open_role < SP_TCP_OPEN_PASSIVE ||
admission->open_role > SP_TCP_OPEN_ACTIVE)
return 0;
if ((admission->policy_role == SP_TCP_ROLE_DIRECT ||
admission->policy_role == SP_TCP_ROLE_SOCKS) &&
admission->needs_connect == 0)
return 0;
if (admission->open_role == SP_TCP_OPEN_PASSIVE)
return admission->passive_syn_present != 0;
return admission->passive_syn_present == 0 && admission->peer_isn == 0 &&
admission->peer_window == 0 &&
admission->peer_mss_present == 0 &&
admission->peer_mss == 0 &&
admission->peer_ws_present == 0 &&
admission->peer_ws == 0;
}
static int sp_open_stream(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
struct sp_fx_open_stream *effect;
if (sp_effect_add_open_stream(core, effects, &effect) < 0)
return -1;
core->connect_effect.id = effect->context.effect.value;
return 0;
}
static int sp_begin_handshake_emit(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
if (core->open_role == SP_TCP_OPEN_PASSIVE) {
core->state = SP_TCP_CORE_SYN_RECEIVED;
core->snd_nxt_high_abs = core->iss_abs + 1;
return sp_emit_control(core, effects,
SP_TCP_CORE_SYN | SP_TCP_CORE_ACK, core->iss_abs,
core->rcv_acked_abs, 1);
}
core->state = SP_TCP_CORE_SYN_SENT;
core->snd_nxt_high_abs = core->iss_abs + 1;
return sp_emit_control(core, effects, SP_TCP_CORE_SYN, core->iss_abs, 0, 1);
}
static enum sp_tcp_core_result
sp_admit(struct sp_tcp_core *core, const struct sp_ev_flow_admitted *admission,
uint64_t now_ns,
struct sp_typed_effect_list *effects) {
if (core->state != SP_TCP_CORE_CLOSED || !sp_admission_valid(admission))
return SP_TCP_CORE_REJECTED;
if (admission->open_role == SP_TCP_OPEN_PASSIVE &&
((admission->peer_mss_present
? admission->peer_mss < SP_TCP_CORE_MIN_MSS
: admission->peer_mss != 0) ||
(!admission->peer_ws_present && admission->peer_ws != 0)))
return SP_TCP_CORE_REJECTED;
core->flow = admission->key.flow.value;
core->generation = admission->key.generation.value;
core->policy_role = admission->policy_role;
core->open_role = admission->open_role;
core->iss_abs = admission->local_iss;
core->data_seq0_abs = core->iss_abs + 1;
core->snd_una_abs = core->iss_abs;
core->snd_nxt_high_abs = core->iss_abs;
core->rcv_wind_shift = admission->local_rcv_wscale;
core->pending_capacity = SP_TCP_CORE_PENDING_CAPACITY;
sp_arm_timer(core, SP_TCP_TIMER_HANDSHAKE,
now_ns + SP_TCP_HANDSHAKE_NS);
if (admission->open_role == SP_TCP_OPEN_PASSIVE) {
if (sp_peer_options_apply(
core, &(struct sp_tcp_peer_options){
.mss = admission->peer_mss,
.mss_present = admission->peer_mss_present,
.ws_present = admission->peer_ws_present,
.ws_raw = admission->peer_ws}, 1) < 0)
return SP_TCP_CORE_REJECTED;
core->irs_abs = admission->peer_isn;
core->rcv_acked_abs = core->irs_abs + 1;
core->rx_pending_end_abs = core->rcv_acked_abs;
core->rx_authorized_end_abs = core->rcv_acked_abs;
core->rcv_adv_right_abs = core->rcv_acked_abs + UINT16_MAX;
core->snd_wnd = admission->peer_window;
}
if (admission->needs_connect != 0) {
core->state = admission->open_role == SP_TCP_OPEN_PASSIVE
? SP_TCP_CORE_CONNECTING_PASSIVE
: SP_TCP_CORE_CONNECTING_ACTIVE;
if (sp_open_stream(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if (sp_begin_handshake_emit(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static int sp_effect_matches(const struct sp_tcp_core *core,
struct sp_flow_key key, sp_effect_id effect,
const struct sp_tcp_live_effect *live) {
return key.flow.value == core->flow &&
key.generation.value == core->generation && live->id != 0 &&
effect.value == live->id;
}
static int sp_close_flow(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
struct sp_fx_close_flow *effect;
memset(&core->connect_effect, 0, sizeof(core->connect_effect));
memset(&core->send_caps, 0, sizeof(core->send_caps));
core->send_cap_count = 0;
memset(&core->stream_consume_effect, 0, sizeof(core->stream_consume_effect));
memset(&core->tun_emit_effect, 0, sizeof(core->tun_emit_effect));
memset(&core->ingress_dispose_effect, 0,
sizeof(core->ingress_dispose_effect));
core->state = SP_TCP_CORE_CLOSED;
/* A terminal transition cancels every deadline so no retransmission or
* probe can be emitted after expiry, including one whose timer event was
* already in flight when the flow closed. */
sp_arm_timer(core, SP_TCP_TIMER_HANDSHAKE, 0);
sp_arm_timer(core, SP_TCP_TIMER_RTO, 0);
sp_arm_timer(core, SP_TCP_TIMER_PERSIST, 0);
sp_arm_timer(core, SP_TCP_TIMER_SEND_AGE, 0);
sp_arm_timer(core, SP_TCP_TIMER_RECEIVE_AGE, 0);
sp_arm_timer(core, SP_TCP_TIMER_CLOSE, 0);
sp_arm_timer(core, SP_TCP_TIMER_TIME_WAIT, 0);
if (sp_effect_add_close_flow(core, effects, &effect) < 0)
return -1;
return 0;
}
static int sp_enter_time_wait(struct sp_tcp_core *core, uint64_t now_ns) {
core->state = SP_TCP_CORE_TIME_WAIT;
sp_arm_timer(core, SP_TCP_TIMER_CLOSE, 0);
sp_arm_timer(core, SP_TCP_TIMER_TIME_WAIT, now_ns + SP_TCP_TIME_WAIT_NS);
return 0;
}
static int sp_maybe_emit_stream_consume(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
struct sp_fx_stream_consume *effect;
if (core->stream_consume_effect.id != 0 ||
core->stream_consumed >= core->stream_acked)
return 0;
if (sp_effect_add_stream_consume(core, effects, &effect) < 0)
return -1;
effect->start.value = core->stream_consumed;
effect->end.value = core->stream_acked;
core->consume_target = effect->end.value;
core->stream_consume_effect.id = effect->context.effect.value;
core->stream_consume_effect.start = effect->start.value;
core->stream_consume_effect.end = effect->end.value;
return 0;
}
static int sp_maybe_emit_to_child(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects) {
struct sp_fx_tcp_emit_batch *effect;
uint64_t allowed;
uint64_t length;
if (core->tun_emit_effect.id != 0 || core->tun_segment_credit == 0 ||
(core->state != SP_TCP_CORE_ESTABLISHED &&
core->state != SP_TCP_CORE_CLOSE_WAIT))
return 0;
if (core->stream_injected < core->stream_tail) {
uint64_t flight_end = core->data_seq0_abs + core->stream_injected;
uint64_t right = core->snd_una_abs + core->snd_wnd;
uint64_t segment_credit;
uint64_t batch_cap;
if (right <= flight_end) {
if (core->persist_deadline_ns == 0)
sp_arm_timer(core, SP_TCP_TIMER_PERSIST,
core->last_now_ns +
sp_persist_delay_ns(core->persist_probes));
return 0;
}
allowed = right - flight_end;
length = sp_min_u64(core->stream_tail - core->stream_injected, allowed);
segment_credit = sp_min_u64(core->tun_segment_credit,
SP_TCP_CORE_MAX_SEGMENTS);
batch_cap = segment_credit * core->effective_send_mss;
length = sp_min_u64(length, batch_cap);
if (length == 0)
return 0;
if (sp_effect_add_tcp_emit_batch(core, effects, &effect) < 0)
return -1;
effect->start.value = core->stream_injected;
effect->end.value = core->stream_injected + length;
effect->seq.value = core->data_seq0_abs + effect->start.value;
effect->ack.value = core->rcv_acked_abs;
effect->flags = SP_TCP_CORE_ACK | SP_TCP_CORE_PSH;
effect->window = sp_current_window(core);
effect->mss = core->effective_send_mss;
effect->terminal = core->stream_eof &&
effect->end.value == core->final_stream_length;
if (effect->terminal)
effect->flags |= SP_TCP_CORE_FIN;
effect->attempt = 1;
core->tun_emit_effect.id = effect->context.effect.value;
core->tun_emit_effect.start = effect->start.value;
core->tun_emit_effect.end = effect->end.value;
return 0;
}
if (core->stream_eof && core->final_stream_length == core->stream_injected &&
core->fin_injected == 0) {
if (sp_emit_control(core, effects, SP_TCP_CORE_FIN | SP_TCP_CORE_ACK,
core->data_seq0_abs + core->stream_injected,
core->rcv_acked_abs, 0) < 0)
return -1;
{
struct sp_fx_tcp_emit_control *control =
&effects->items[effects->count - 1].value.tcp_emit_control;
control->terminal = 1;
core->tun_emit_effect.id = control->context.effect.value;
}
core->tun_emit_effect.start = core->stream_injected;
core->tun_emit_effect.end = core->stream_injected;
}
return 0;
}
/* Rebuilds a retransmission batch from the oldest unacknowledged byte, bounded
* by cap_bytes. The RTO path passes a full authorized batch; the bounded
* fast retransmit passes exactly one MSS. Returns 1 when an effect was
* emitted, 0 when there was nothing to retransmit, and -1 on effect overflow. */
static int sp_emit_retransmit(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects,
uint64_t cap_bytes, uint32_t attempt) {
struct sp_fx_tcp_emit_batch *effect;
if (core->tun_emit_effect.id != 0 ||
core->stream_acked >= core->stream_injected)
return 0;
if (sp_effect_add_tcp_emit_batch(core, effects, &effect) < 0)
return -1;
effect->start.value = core->stream_acked;
effect->end.value =
sp_min_u64(core->stream_injected, core->stream_acked + cap_bytes);
effect->seq.value = core->data_seq0_abs + effect->start.value;
effect->ack.value = core->rcv_acked_abs;
effect->flags = SP_TCP_CORE_ACK | SP_TCP_CORE_PSH;
effect->window = sp_current_window(core);
effect->mss = core->effective_send_mss;
effect->terminal =
core->fin_injected && effect->end.value == core->final_stream_length;
if (effect->terminal)
effect->flags |= SP_TCP_CORE_FIN;
effect->attempt = attempt;
core->tun_emit_effect.id = effect->context.effect.value;
core->tun_emit_effect.start = effect->start.value;
core->tun_emit_effect.end = effect->end.value;
core->tun_emit_effect.retransmission = 1;
return 1;
}
static int sp_ack_update(struct sp_tcp_core *core,
const struct sp_ev_tcp_segment *segment,
uint64_t seq_abs,
struct sp_typed_effect_list *effects) {
uint64_t ack_abs;
uint64_t offered_window;
int window_changed;
if ((segment->flags & SP_TCP_CORE_ACK) == 0)
return 0;
ack_abs = sp_tcp_unwrap_u32(segment->ack, core->snd_una_abs);
if (ack_abs > core->snd_nxt_high_abs) {
if (sp_emit_current_ack(core, effects) < 0)
return -1;
return 1;
}
offered_window = segment->window;
if (core->snd_ws_ok)
offered_window <<= core->snd_wind_shift;
window_changed = offered_window != core->snd_wnd;
if (ack_abs >= core->snd_una_abs) {
if (ack_abs > core->snd_una_abs) {
if (core->rtt_sample_end_abs != 0 &&
ack_abs >= core->rtt_sample_end_abs &&
!core->rtt_sample_ambiguous &&
core->last_now_ns >= core->rtt_sample_sent_ns) {
uint64_t sample = core->last_now_ns - core->rtt_sample_sent_ns;
if (core->rtt_samples == 0) {
core->srtt_ns = sample;
core->rttvar_ns = sample / 2;
} else {
uint64_t delta = core->srtt_ns > sample
? core->srtt_ns - sample
: sample - core->srtt_ns;
core->rttvar_ns = (3 * core->rttvar_ns + delta) / 4;
core->srtt_ns = (7 * core->srtt_ns + sample) / 8;
}
core->rtt_samples++;
core->rtt_sample_end_abs = 0;
core->rtt_sample_sent_ns = 0;
core->rtt_sample_ambiguous = 0;
}
core->snd_una_abs = ack_abs;
/* A valid cumulative ACK restarts the RTO for the new oldest
* obligation, but the backed-off interval and consecutive-expiry count
* collapse only once unambiguously transmitted new data beyond the
* recovery point is acknowledged. A partial ACK of retransmitted data
* leaves the backoff history intact. */
if (core->rto_recovery_point_abs == 0 ||
ack_abs >= core->rto_recovery_point_abs) {
core->rto_retries = 0;
core->rto_first_ns = 0;
core->rto_recovery_point_abs = 0;
}
/* Cumulative progress ends the current fast-retransmit episode. */
core->dup_ack_count = 0;
core->dup_ack_ref_abs = ack_abs;
if (core->fast_retransmit_point_abs != 0 &&
ack_abs >= core->fast_retransmit_point_abs) {
core->fast_retransmit_point_abs = 0;
core->fast_retransmit_used = 0;
}
if (ack_abs > core->data_seq0_abs) {
uint64_t data_ack = ack_abs - core->data_seq0_abs;
if (data_ack > core->stream_injected)
data_ack = core->stream_injected;
if (data_ack > core->stream_acked)
core->stream_acked = data_ack;
}
if (core->fin_injected &&
ack_abs >= core->data_seq0_abs + core->stream_injected + 1)
core->fin_acked = 1;
sp_arm_timer(core, SP_TCP_TIMER_RTO,
core->snd_una_abs < core->snd_nxt_high_abs
? core->last_now_ns +
sp_rto_delay_ns(core, core->rto_retries)
: 0);
sp_update_send_age(core, core->last_now_ns, 1);
if (sp_maybe_emit_stream_consume(core, effects) < 0)
return -1;
} else if (segment->payload_length.value == 0 && !window_changed &&
(segment->flags & (SP_TCP_CORE_SYN | SP_TCP_CORE_FIN)) == 0 &&
core->snd_una_abs < core->snd_nxt_high_abs) {
/* Bounded one-shot fast retransmit. Only a pure, window-neutral
* duplicate of the current SND.UNA qualifies; data-bearing duplicates,
* window-only updates, and challenge/unacceptable traffic are excluded
* by the guards above and by the acceptability check upstream. Three
* qualifying duplicates retransmit exactly the oldest MSS once per
* recovery point. The RTO stays armed and no congestion window is
* maintained on the private child link. */
if (core->dup_ack_ref_abs != ack_abs) {
core->dup_ack_ref_abs = ack_abs;
core->dup_ack_count = 0;
}
if (core->dup_ack_count < UINT32_MAX)
core->dup_ack_count++;
if (core->dup_ack_count == SP_TCP_DUP_ACK_THRESHOLD &&
!core->fast_retransmit_used) {
int emitted =
sp_emit_retransmit(core, effects, core->effective_send_mss, 1);
if (emitted < 0)
return -1;
if (emitted == 1) {
core->fast_retransmit_used = 1;
core->fast_retransmit_point_abs = core->snd_nxt_high_abs;
core->rtt_sample_ambiguous = 1;
}
}
}
if (seq_abs > core->snd_wl1_abs ||
(seq_abs == core->snd_wl1_abs && ack_abs >= core->snd_wl2_abs)) {
uint64_t window = segment->window;
if (core->snd_ws_ok)
window <<= core->snd_wind_shift;
core->snd_wnd = window;
core->snd_wl1_abs = seq_abs;
core->snd_wl2_abs = ack_abs;
if (window != 0) {
sp_arm_timer(core, SP_TCP_TIMER_PERSIST, 0);
core->persist_probes = 0;
}
}
}
return 0;
}
static int sp_segment_acceptable(const struct sp_tcp_core *core,
uint64_t seq_abs, uint64_t length) {
if (length == 0) {
if (core->rcv_adv_right_abs == core->rcv_acked_abs)
return seq_abs == core->rcv_acked_abs;
return seq_abs >= core->rcv_acked_abs &&
seq_abs <= core->rcv_adv_right_abs;
}
return seq_abs < core->rcv_adv_right_abs &&
seq_abs + length > core->rcv_acked_abs;
}
/* Append one exact-slice STREAM_SEND capability to the per-flow FIFO. The FIFO
* holds up to K_rx live capabilities tiling [A, Q) in ascending sequence order:
* fresh authorizations are appended at [Q, end) and, once the FIFO is empty, a
* durable-reservation re-flush is appended at [A, P). Returns 1 when the FIFO
* is full, 0 on success, -1 on effect
* overflow. */
static int sp_stream_send_effect(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects,
uint64_t start, uint64_t end,
uint64_t ingress_id) {
struct sp_fx_stream_send *effect;
struct sp_tcp_live_effect *cap;
if (core->send_cap_count >= SP_TCP_CORE_K_RX)
return 1;
if (sp_effect_add_stream_send(core, effects, &effect) < 0)
return -1;
effect->start.value = start;
effect->end.value = end;
effect->capability.value = ingress_id;
cap = &core->send_caps[core->send_cap_count++];
memset(cap, 0, sizeof(*cap));
cap->id = effect->context.effect.value;
cap->start = start;
cap->end = end;
cap->ingress_id = ingress_id;
return 0;
}
static int sp_handle_peer_fin(struct sp_tcp_core *core,
struct sp_typed_effect_list *effects,
uint64_t fin_abs, uint64_t now_ns) {
struct sp_fx_stream_shutdown *effect;
if (fin_abs != core->rcv_acked_abs ||
core->rx_pending_end_abs != core->rcv_acked_abs ||
core->rx_authorized_end_abs != core->rcv_acked_abs) {
core->peer_fin_seen = 1;
core->peer_fin_abs = fin_abs;
return sp_emit_current_ack(core, effects);
}
core->peer_fin_seen = 1;
core->peer_fin_abs = fin_abs;
core->rcv_acked_abs++;
core->rx_pending_end_abs++;
core->rx_authorized_end_abs++;
if (core->rcv_adv_right_abs < core->rcv_acked_abs)
core->rcv_adv_right_abs = core->rcv_acked_abs;
if (sp_emit_current_ack(core, effects) < 0)
return -1;
if (sp_effect_add_stream_shutdown(core, effects, &effect) < 0)
return -1;
if (core->state == SP_TCP_CORE_ESTABLISHED)
core->state = SP_TCP_CORE_CLOSE_WAIT;
else if (core->state == SP_TCP_CORE_FIN_WAIT_1)
core->state = core->fin_acked ? SP_TCP_CORE_TIME_WAIT : SP_TCP_CORE_CLOSING;
else if (core->state == SP_TCP_CORE_FIN_WAIT_2)
sp_enter_time_wait(core, now_ns);
return 0;
}
static enum sp_tcp_core_result
sp_handle_segment(struct sp_tcp_core *core,
const struct sp_ev_tcp_segment *segment, uint64_t now_ns,
struct sp_typed_effect_list *effects) {
uint64_t seq_abs;
uint64_t sequence_len = segment->payload_length.value;
uint64_t data_start;
uint64_t data_end;
int ack_result;
if ((segment->flags & SP_TCP_CORE_SYN) != 0)
sequence_len++;
if ((segment->flags & SP_TCP_CORE_FIN) != 0)
sequence_len++;
if (core->state == SP_TCP_CORE_SYN_RECEIVED) {
if ((segment->flags & SP_TCP_CORE_RST) != 0) {
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if ((segment->flags & SP_TCP_CORE_SYN) != 0 &&
segment->seq == sp_tcp_wire(core->irs_abs)) {
if (sp_emit_control(core, effects,
SP_TCP_CORE_SYN | SP_TCP_CORE_ACK, core->iss_abs,
core->rcv_acked_abs, 1) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if ((segment->flags & SP_TCP_CORE_ACK) != 0 &&
segment->ack == sp_tcp_wire(core->iss_abs + 1) &&
segment->seq == sp_tcp_wire(core->rcv_acked_abs)) {
core->snd_una_abs = core->iss_abs + 1;
core->snd_wl1_abs = core->rcv_acked_abs;
core->snd_wl2_abs = core->snd_una_abs;
core->snd_wnd = (uint64_t)segment->window << core->snd_wind_shift;
core->state = SP_TCP_CORE_ESTABLISHED;
sp_arm_timer(core, SP_TCP_TIMER_HANDSHAKE, 0);
return SP_TCP_CORE_OK;
}
return SP_TCP_CORE_REJECTED;
}
if (core->state == SP_TCP_CORE_SYN_SENT) {
if ((segment->flags & (SP_TCP_CORE_SYN | SP_TCP_CORE_ACK)) ==
(SP_TCP_CORE_SYN | SP_TCP_CORE_ACK) &&
segment->ack == sp_tcp_wire(core->iss_abs + 1) &&
sp_peer_options_apply(
core, &(struct sp_tcp_peer_options){
.mss = segment->mss,
.mss_present = segment->mss_present,
.ws_present = segment->ws_present,
.ws_raw = segment->ws}, 0) == 0) {
core->irs_abs = segment->seq;
core->rcv_acked_abs = core->irs_abs + 1;
core->rx_pending_end_abs = core->rcv_acked_abs;
core->rx_authorized_end_abs = core->rcv_acked_abs;
core->rcv_adv_right_abs = core->rcv_acked_abs + UINT16_MAX;
core->snd_una_abs = core->iss_abs + 1;
core->snd_wnd = segment->window;
core->state = SP_TCP_CORE_ESTABLISHED;
sp_arm_timer(core, SP_TCP_TIMER_HANDSHAKE, 0);
if (sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if ((segment->flags & SP_TCP_CORE_SYN) != 0 &&
(segment->flags & SP_TCP_CORE_ACK) == 0) {
if (sp_emit_control(core, effects, SP_TCP_CORE_RST, core->iss_abs, 0, 0) <
0 ||
sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
return SP_TCP_CORE_REJECTED;
}
if (core->state < SP_TCP_CORE_ESTABLISHED ||
core->state > SP_TCP_CORE_TIME_WAIT)
return SP_TCP_CORE_REJECTED;
seq_abs = sp_tcp_unwrap_u32(segment->seq, core->rcv_acked_abs);
if (!sp_segment_acceptable(core, seq_abs, sequence_len)) {
if ((segment->flags & SP_TCP_CORE_RST) == 0 &&
sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (sp_emit_ingress_dispose(core, effects, segment->ingress.value) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if ((segment->flags & SP_TCP_CORE_RST) != 0) {
if (seq_abs == core->rcv_acked_abs) {
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
} else if (sp_emit_current_ack(core, effects) < 0) {
return SP_TCP_CORE_EFFECT_OVERFLOW;
}
return SP_TCP_CORE_OK;
}
if ((segment->flags & SP_TCP_CORE_SYN) != 0) {
if (sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
ack_result = sp_ack_update(core, segment, seq_abs, effects);
if (ack_result < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (ack_result > 0)
return SP_TCP_CORE_OK;
data_start = seq_abs;
data_end = seq_abs + segment->payload_length.value;
if (segment->payload_length.value != 0) {
if (data_end <= core->rx_authorized_end_abs) {
if (sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
} else if (data_start > core->rx_authorized_end_abs) {
if (sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
} else if (core->send_cap_count < SP_TCP_CORE_K_RX) {
if (data_start < core->rx_authorized_end_abs)
data_start = core->rx_authorized_end_abs;
if (data_end > core->rcv_adv_right_abs)
data_end = core->rcv_adv_right_abs;
if (data_end > data_start) {
core->rx_authorized_end_abs = data_end;
sp_update_receive_age(core, now_ns, 0);
if (sp_stream_send_effect(core, effects, data_start, data_end,
segment->ingress.value) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
}
}
}
if ((segment->flags & SP_TCP_CORE_FIN) != 0) {
uint64_t fin_abs = seq_abs + segment->payload_length.value;
if (fin_abs < core->rcv_adv_right_abs &&
sp_handle_peer_fin(core, effects, fin_abs, now_ns) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
}
if (segment->payload_length.value == 0 &&
(segment->flags & SP_TCP_CORE_FIN) == 0 &&
sp_emit_ingress_dispose(core, effects, segment->ingress.value) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (core->fin_acked) {
if (core->state == SP_TCP_CORE_FIN_WAIT_1)
core->state = SP_TCP_CORE_FIN_WAIT_2;
else if (core->state == SP_TCP_CORE_CLOSING)
sp_enter_time_wait(core, now_ns);
else if (core->state == SP_TCP_CORE_LAST_ACK &&
sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
}
if (sp_maybe_emit_to_child(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_connect_result(struct sp_tcp_core *core,
const struct sp_ev_connect_result *input,
struct sp_typed_effect_list *effects) {
if (!sp_effect_matches(core, input->context.key, input->effect,
&core->connect_effect))
return SP_TCP_CORE_OK;
memset(&core->connect_effect, 0, sizeof(core->connect_effect));
if (input->outcome == SP_CONNECT_FAILED) {
if (core->open_role == SP_TCP_OPEN_PASSIVE &&
sp_emit_control(core, effects, SP_TCP_CORE_RST, core->iss_abs, 0, 0) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
if (sp_begin_handshake_emit(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_stream_send_result(struct sp_tcp_core *core,
const struct sp_ev_stream_send_result *input,
struct sp_typed_effect_list *effects) {
struct sp_tcp_live_effect *head;
uint64_t length;
uint64_t accepted;
uint64_t retained;
uint64_t ingress_id;
/* Completions linearize FIFO: only the oldest capability (the head, whose
* start is the ACK frontier A for a re-flush or the durable frontier P for a
* fresh slice) may commit. The executor processes a flow's capabilities in
* strict order, so a result naming a still-live non-head capability is an
* ordering violation and fails loud; any id no longer in the FIFO is a stale
* completion and is ignored. */
if (input->context.key.flow.value != core->flow ||
input->context.key.generation.value != core->generation ||
core->send_cap_count == 0)
return SP_TCP_CORE_OK;
head = &core->send_caps[0];
if (input->effect.value != head->id) {
for (unsigned i = 1; i < core->send_cap_count; ++i)
if (core->send_caps[i].id == input->effect.value)
return SP_TCP_CORE_BAD_EVENT;
return SP_TCP_CORE_OK;
}
length = head->end - head->start;
accepted = input->accepted.value;
retained = input->retained.value;
/* accepted may advance A only through a prefix contiguous with A; the executor
* never accepts bytes past a durable gap, so accepted > 0 requires the head to
* start exactly at A. The whole slice must be accounted (accepted+retained). */
if (input->outcome == SP_SEND_FAILED || accepted > length ||
retained > length - accepted || accepted + retained != length ||
(accepted != 0 && head->start != core->rcv_acked_abs))
return SP_TCP_CORE_BAD_EVENT;
ingress_id = head->ingress_id;
core->rcv_acked_abs += accepted;
core->rx_pending_end_abs = head->start + accepted + retained;
sp_update_receive_age(core, input->context.now.value, accepted != 0);
core->send_cap_count--;
if (core->send_cap_count > 0)
memmove(&core->send_caps[0], &core->send_caps[1],
(size_t)core->send_cap_count * sizeof(core->send_caps[0]));
memset(&core->send_caps[core->send_cap_count], 0,
sizeof(core->send_caps[0]));
if (core->send_cap_count == 0)
core->rx_authorized_end_abs = core->rx_pending_end_abs;
if (accepted != 0 && sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (ingress_id != 0 &&
sp_emit_ingress_dispose(core, effects, ingress_id) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (core->peer_fin_seen && core->peer_fin_abs == core->rcv_acked_abs &&
core->rx_pending_end_abs == core->rcv_acked_abs &&
sp_handle_peer_fin(core, effects, core->peer_fin_abs,
input->context.now.value) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_stream_consumed(struct sp_tcp_core *core,
const struct sp_ev_stream_consumed *input,
struct sp_typed_effect_list *effects) {
uint64_t length;
uint64_t accepted;
if (!sp_effect_matches(core, input->context.key, input->effect,
&core->stream_consume_effect))
return SP_TCP_CORE_OK;
length = core->stream_consume_effect.end - core->stream_consume_effect.start;
accepted = input->accepted.value;
if (accepted > length ||
core->stream_consume_effect.start != core->stream_consumed)
return SP_TCP_CORE_BAD_EVENT;
if (accepted == 0) {
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
core->stream_consumed += accepted;
core->consume_target = core->stream_consumed;
memset(&core->stream_consume_effect, 0, sizeof(core->stream_consume_effect));
if (sp_maybe_emit_stream_consume(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_tun_emit_result(struct sp_tcp_core *core,
const struct sp_ev_tun_emit_result *input,
struct sp_typed_effect_list *effects) {
uint64_t length;
uint64_t accepted;
uint64_t segments;
if (!sp_effect_matches(core, input->context.key, input->effect,
&core->tun_emit_effect))
return SP_TCP_CORE_OK;
length = core->tun_emit_effect.end - core->tun_emit_effect.start;
accepted = input->accepted.value;
if (input->outcome == SP_EMIT_FAILED || accepted > length ||
(length != 0 && accepted == 0) ||
(accepted < length &&
accepted % core->effective_send_mss != 0))
return SP_TCP_CORE_BAD_EVENT;
if (core->tun_emit_effect.retransmission) {
memset(&core->tun_emit_effect, 0, sizeof(core->tun_emit_effect));
return SP_TCP_CORE_OK;
}
if (core->tun_emit_effect.start != core->stream_injected)
return SP_TCP_CORE_BAD_EVENT;
if (length != 0 &&
(accepted + core->effective_send_mss - 1) / core->effective_send_mss >
core->tun_segment_credit)
return SP_TCP_CORE_BAD_EVENT;
if (length == 0) {
core->fin_injected = 1;
core->snd_nxt_high_abs++;
core->state = core->state == SP_TCP_CORE_CLOSE_WAIT
? SP_TCP_CORE_LAST_ACK
: SP_TCP_CORE_FIN_WAIT_1;
} else {
segments = (accepted + core->effective_send_mss - 1) /
core->effective_send_mss;
core->stream_injected += accepted;
if (accepted == length && core->stream_eof &&
core->stream_injected == core->final_stream_length) {
core->fin_injected = 1;
core->state = core->state == SP_TCP_CORE_CLOSE_WAIT
? SP_TCP_CORE_LAST_ACK
: SP_TCP_CORE_FIN_WAIT_1;
}
core->snd_nxt_high_abs = core->data_seq0_abs + core->stream_injected +
(core->fin_injected ? 1 : 0);
core->tun_segment_credit -= segments;
sp_arm_timer(core, SP_TCP_TIMER_RTO,
input->context.now.value +
sp_rto_delay_ns(core, core->rto_retries));
sp_update_send_age(core, input->context.now.value, 0);
if (core->rtt_sample_end_abs == 0) {
core->rtt_sample_end_abs = core->data_seq0_abs + core->stream_injected;
core->rtt_sample_sent_ns = input->context.now.value;
core->rtt_sample_ambiguous = 0;
}
}
memset(&core->tun_emit_effect, 0, sizeof(core->tun_emit_effect));
if (sp_maybe_emit_to_child(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_credit(struct sp_tcp_core *core, const struct sp_ev_credit *input,
struct sp_typed_effect_list *effects) {
uint64_t requested_capacity = input->pending_capacity.value;
if (input->stream_tail.value < core->stream_tail ||
input->stream_tail.value - core->stream_consumed > SP_TCP_CORE_PENDING_CAPACITY ||
(core->stream_eof && input->stream_tail.value != core->final_stream_length) ||
input->stream_tail.value > UINT64_MAX - core->data_seq0_abs - 1 ||
requested_capacity > SP_TCP_CORE_PENDING_CAPACITY)
return SP_TCP_CORE_BAD_EVENT;
if (requested_capacity != 0) {
uint64_t scale =
core->snd_ws_ok ? UINT64_C(1) << core->rcv_wind_shift : 1;
uint64_t advertisable_capacity =
(requested_capacity / scale) * scale;
if (core->rcv_adv_right_abs < core->rcv_acked_abs ||
core->rcv_adv_right_abs - core->rcv_acked_abs >
advertisable_capacity)
return SP_TCP_CORE_BAD_EVENT;
}
core->stream_tail = input->stream_tail.value;
if (requested_capacity != 0)
core->pending_capacity = requested_capacity;
if (UINT64_MAX - core->tun_segment_credit < input->tun_segments.value)
core->tun_segment_credit = UINT64_MAX;
else
core->tun_segment_credit += input->tun_segments.value;
if (core->rx_pending_end_abs > core->rcv_acked_abs &&
core->send_cap_count == 0 &&
sp_stream_send_effect(core, effects, core->rcv_acked_abs,
core->rx_pending_end_abs, 0) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (sp_maybe_emit_to_child(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_stream_eof(struct sp_tcp_core *core, const struct sp_ev_stream_eof *input,
struct sp_typed_effect_list *effects) {
if (input->stream_tail.value < core->stream_tail || core->stream_eof ||
input->stream_tail.value - core->stream_consumed > SP_TCP_CORE_PENDING_CAPACITY ||
input->stream_tail.value > UINT64_MAX - core->data_seq0_abs - 1)
return SP_TCP_CORE_BAD_EVENT;
core->stream_tail = input->stream_tail.value;
core->final_stream_length = core->stream_tail;
core->stream_eof = 1;
sp_arm_timer(core, SP_TCP_TIMER_CLOSE,
input->context.now.value + SP_TCP_CLOSE_NS);
if (sp_maybe_emit_to_child(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result
sp_timer(struct sp_tcp_core *core, const struct sp_ev_timer_fired *input,
struct sp_typed_effect_list *effects) {
if (input->deadline.value > input->context.now.value)
return SP_TCP_CORE_OK;
switch (input->kind) {
case SP_TCP_TIMER_HANDSHAKE:
case SP_TCP_TIMER_SEND_AGE:
case SP_TCP_TIMER_RECEIVE_AGE:
case SP_TCP_TIMER_CLOSE:
case SP_TCP_TIMER_TIME_WAIT:
if (!sp_timer_current(core, input->kind, input->deadline.value,
input->timer_generation.value))
return SP_TCP_CORE_OK;
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
case SP_TCP_TIMER_RTO:
if (!sp_timer_current(core, input->kind, input->deadline.value,
input->timer_generation.value))
return SP_TCP_CORE_OK;
if (core->rto_first_ns == 0)
core->rto_first_ns = input->context.now.value;
/* The recovery policy permits at most eight RTO expiries for one unadvanced
* SND.UNA and
* closes on the eighth. rto_retries counts expiries already processed, so
* this expiry is the (rto_retries + 1)th. */
if (core->rto_retries + 1 >= SP_TCP_CORE_MAX_RTO_RETRIES ||
input->context.now.value - core->rto_first_ns >=
SP_TCP_RTO_MAX_AGE_NS) {
if (sp_close_flow(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
core->rto_retries++;
/* The recovery point is the highest sequence transmitted when loss
* was first detected in this episode. Backoff history survives until an
* ACK passes it. */
if (core->rto_recovery_point_abs == 0)
core->rto_recovery_point_abs = core->snd_nxt_high_abs;
core->rtt_sample_ambiguous = 1;
sp_arm_timer(core, SP_TCP_TIMER_RTO,
input->context.now.value +
sp_rto_delay_ns(core, core->rto_retries));
{
int emitted = sp_emit_retransmit(
core, effects,
(uint64_t)SP_TCP_CORE_MAX_SEGMENTS * core->effective_send_mss,
core->rto_retries + 1);
if (emitted < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (emitted == 1)
return SP_TCP_CORE_OK;
}
if (core->tun_emit_effect.id == 0 && core->fin_injected &&
!core->fin_acked) {
struct sp_fx_tcp_emit_control *effect;
if (sp_emit_control(core, effects, SP_TCP_CORE_FIN | SP_TCP_CORE_ACK,
core->data_seq0_abs + core->stream_injected,
core->rcv_acked_abs, 0) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
effect = &effects->items[effects->count - 1].value.tcp_emit_control;
effect->terminal = 1;
effect->attempt = core->rto_retries + 1;
core->tun_emit_effect.id = effect->context.effect.value;
core->tun_emit_effect.start = core->stream_injected;
core->tun_emit_effect.end = core->stream_injected;
core->tun_emit_effect.retransmission = 1;
}
return SP_TCP_CORE_OK;
case SP_TCP_TIMER_PERSIST:
if (!sp_timer_current(core, input->kind, input->deadline.value,
input->timer_generation.value))
return SP_TCP_CORE_OK;
if (core->persist_probes < UINT32_MAX)
core->persist_probes++;
sp_arm_timer(core, SP_TCP_TIMER_PERSIST,
input->context.now.value +
sp_persist_delay_ns(core->persist_probes));
if (sp_emit_current_ack(core, effects) < 0)
return SP_TCP_CORE_EFFECT_OVERFLOW;
return SP_TCP_CORE_OK;
}
return SP_TCP_CORE_BAD_EVENT;
}
int sp_tcp_assert_invariants(const struct sp_tcp_core *core) {
const struct sp_tcp_live_effect *live[] = {
&core->connect_effect,
&core->stream_consume_effect,
&core->tun_emit_effect,
&core->ingress_dispose_effect,
};
if (core->send_cap_count > SP_TCP_CORE_K_RX)
return 0;
if (core->state == SP_TCP_CORE_CLOSED) {
if (core->send_cap_count != 0)
return 0;
for (size_t i = 0; i < sizeof(live) / sizeof(live[0]); ++i)
if (live[i]->id != 0)
return 0;
return 1;
}
if (core->flow == 0 || core->generation == 0 ||
core->policy_role < SP_TCP_ROLE_DIRECT ||
core->policy_role > SP_TCP_ROLE_PUBLISH ||
core->open_role < SP_TCP_OPEN_PASSIVE ||
core->open_role > SP_TCP_OPEN_ACTIVE || core->snd_wind_shift > 14 ||
core->rcv_wind_shift > 14)
return 0;
if (core->state >= SP_TCP_CORE_SYN_RECEIVED &&
core->state != SP_TCP_CORE_SYN_SENT &&
(core->effective_send_mss < SP_TCP_CORE_MIN_MSS ||
core->effective_send_mss > SP_TCP_CORE_MAX_MSS))
return 0;
if (!(core->stream_consumed <= core->consume_target &&
core->consume_target <= core->stream_acked &&
core->stream_acked <= core->stream_injected &&
core->stream_injected <= core->stream_tail))
return 0;
if (core->stream_tail - core->stream_consumed >
SP_TCP_CORE_PENDING_CAPACITY)
return 0;
if ((core->stream_eof && core->final_stream_length != core->stream_tail) ||
(!core->stream_eof && core->final_stream_length != 0))
return 0;
if (!(core->rcv_acked_abs <= core->rx_pending_end_abs &&
core->rx_pending_end_abs <= core->rx_authorized_end_abs &&
core->rx_authorized_end_abs <= core->rcv_adv_right_abs))
return 0;
{
uint64_t scale =
core->snd_ws_ok ? UINT64_C(1) << core->rcv_wind_shift : 1;
uint64_t advertisable_capacity =
(core->pending_capacity / scale) * scale;
if (advertisable_capacity == 0 ||
UINT64_MAX - core->rcv_acked_abs < advertisable_capacity ||
core->rcv_adv_right_abs >
core->rcv_acked_abs + advertisable_capacity)
return 0;
}
if (core->snd_una_abs > core->snd_nxt_high_abs)
return 0;
if (core->state >= SP_TCP_CORE_ESTABLISHED &&
core->snd_nxt_high_abs !=
core->data_seq0_abs + core->stream_injected +
(core->fin_injected ? 1U : 0U))
return 0;
if (core->fin_acked && !core->fin_injected)
return 0;
/* The STREAM_SEND capability FIFO tiles [caps[0].start, Q) contiguously in
* ascending sequence, with A <= caps[0].start <= P and caps[last].end == Q;
* an empty FIFO means no authorization outstanding (Q == P). */
if (core->send_cap_count == 0) {
if (core->rx_authorized_end_abs != core->rx_pending_end_abs)
return 0;
} else {
const struct sp_tcp_live_effect *first = &core->send_caps[0];
const struct sp_tcp_live_effect *last =
&core->send_caps[core->send_cap_count - 1];
if (first->start < core->rcv_acked_abs ||
first->start > core->rx_pending_end_abs ||
last->end != core->rx_authorized_end_abs)
return 0;
for (unsigned i = 0; i < core->send_cap_count; ++i) {
const struct sp_tcp_live_effect *cap = &core->send_caps[i];
if (cap->id == 0 || cap->start >= cap->end)
return 0;
if (i + 1 < core->send_cap_count &&
cap->end != core->send_caps[i + 1].start)
return 0;
}
}
/* Live-effect and capability ids are pairwise distinct. */
for (size_t i = 0; i < sizeof(live) / sizeof(live[0]); ++i) {
if (live[i]->id == 0)
continue;
for (size_t j = i + 1; j < sizeof(live) / sizeof(live[0]); ++j)
if (live[i]->id == live[j]->id)
return 0;
for (unsigned k = 0; k < core->send_cap_count; ++k)
if (live[i]->id == core->send_caps[k].id)
return 0;
}
for (unsigned i = 0; i < core->send_cap_count; ++i)
for (unsigned j = i + 1; j < core->send_cap_count; ++j)
if (core->send_caps[i].id == core->send_caps[j].id)
return 0;
return 1;
}
SP_TEST_API static enum sp_construct_result
sp_typed_effect_construct_check(const struct sp_typed_effect *effect) {
const struct sp_typed_effect_context *context =
sp_typed_effect_context(effect);
if (context == NULL || context->effect.value == 0 ||
context->key.flow.value == 0 || context->key.generation.value == 0)
return SP_CONSTRUCT_INTERNAL_INVARIANT;
switch (effect->type) {
case SP_FX_OPEN_STREAM:
return SP_CONSTRUCT_OK;
case SP_FX_TCP_EMIT_BATCH: {
const struct sp_fx_tcp_emit_batch *v = &effect->value.tcp_emit_batch;
return v->end.value >= v->start.value && v->terminal <= 1
? SP_CONSTRUCT_OK
: SP_CONSTRUCT_INTERNAL_INVARIANT;
}
case SP_FX_TCP_EMIT_CONTROL: {
const struct sp_fx_tcp_emit_control *v = &effect->value.tcp_emit_control;
return v->ws_present <= 1 && v->ws <= 14 && v->terminal <= 1
? SP_CONSTRUCT_OK
: SP_CONSTRUCT_INTERNAL_INVARIANT;
}
case SP_FX_STREAM_SEND:
return effect->value.stream_send.end.value >=
effect->value.stream_send.start.value
? SP_CONSTRUCT_OK
: SP_CONSTRUCT_INTERNAL_INVARIANT;
case SP_FX_STREAM_CONSUME:
return effect->value.stream_consume.end.value >=
effect->value.stream_consume.start.value
? SP_CONSTRUCT_OK
: SP_CONSTRUCT_INTERNAL_INVARIANT;
case SP_FX_INGRESS_DISPOSE:
return effect->value.ingress_dispose.ingress.value != 0
? SP_CONSTRUCT_OK
: SP_CONSTRUCT_INTERNAL_INVARIANT;
case SP_FX_STREAM_SHUTDOWN:
case SP_FX_CLOSE_FLOW:
return SP_CONSTRUCT_OK;
}
return SP_CONSTRUCT_INTERNAL_INVARIANT;
}
/* Family-specific pure-core entry points are the semantic API. Every caller
* supplies one exact event product and receives exact effect products. */
/* Family handlers reject invalid inputs before semantic mutation. The only
* tentative common metadata is the injected time and legacy fixture sequence.
* No whole-flow snapshot is taken on the packet path. */
struct sp_core_transaction {
uint64_t now;
uint64_t control_sequence;
};
static enum sp_tcp_core_result sp_core_transaction_begin(
struct sp_tcp_core *core, const struct sp_typed_event_context *context,
struct sp_typed_effect_list *effects, struct sp_core_transaction *transaction,
int *apply) {
*apply = 0;
if (context->key.flow.value == 0 ||
context->key.generation.value == 0)
return SP_TCP_CORE_BAD_EVENT;
transaction->now = core->last_now_ns;
transaction->control_sequence = core->applied_control_seq;
/* Only items below count are output; sp_effect_reserve() clears each slot
* as it is produced, so unused slots are never written (O9). */
effects->count = 0;
if (context->key.flow.value != core->flow ||
context->key.generation.value != core->generation)
return SP_TCP_CORE_OK;
if (context->now.value < core->last_now_ns)
return SP_TCP_CORE_REJECTED;
/* At most eight effects can be produced by a typed turn: ACK/consume,
* send, peer-FIN ACK/shutdown, disposal/close and one new emit. Reserve
* the complete fixed output batch and ID range before any mutation. */
if (core->next_effect_id > UINT64_MAX - SP_TCP_CORE_MAX_EFFECTS)
return SP_TCP_CORE_EFFECT_OVERFLOW;
if (context->causal_fence.value > core->applied_control_seq)
return SP_TCP_CORE_BAD_EVENT;
if (context->control_sequence.value != 0) {
if (core->applied_control_seq == UINT64_MAX ||
context->control_sequence.value != core->applied_control_seq + 1)
return SP_TCP_CORE_BAD_EVENT;
core->applied_control_seq = context->control_sequence.value;
}
core->last_now_ns = context->now.value;
*apply = 1;
return SP_TCP_CORE_OK;
}
static enum sp_tcp_core_result sp_core_transaction_finish(
struct sp_tcp_core *core, struct sp_typed_effect_list *effects,
const struct sp_core_transaction *transaction,
enum sp_tcp_core_result result) {
if (result != SP_TCP_CORE_OK) {
core->last_now_ns = transaction->now;
core->applied_control_seq = transaction->control_sequence;
effects->count = 0;
}
#ifdef SOCKPUPPET_CORE_CHECKS
assert(sp_tcp_assert_invariants(core));
for (size_t i = 0; i < effects->count; ++i)
assert(sp_typed_effect_construct_check(&effects->items[i]) == SP_CONSTRUCT_OK);
#endif
return result;
}
enum sp_tcp_core_result
sp_tcp_core_admit(struct sp_tcp_core *core,
const struct sp_ev_flow_admitted *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction = {
.now = core->last_now_ns, .control_sequence = core->applied_control_seq};
enum sp_tcp_core_result result;
effects->count = 0; /* see sp_core_transaction_begin() */
if (core->next_effect_id > UINT64_MAX - SP_TCP_CORE_MAX_EFFECTS)
return SP_TCP_CORE_EFFECT_OVERFLOW;
core->last_now_ns = input->now.value;
result = sp_admit(core, input, input->now.value, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_segment(struct sp_tcp_core *core,
const struct sp_ev_tcp_segment *input,
struct sp_typed_effect_list *effects) {
if (input->payload_length.value > UINT32_MAX)
return SP_TCP_CORE_BAD_EVENT;
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_handle_segment(core, input, input->context.now.value,
effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_connect_result(struct sp_tcp_core *core,
const struct sp_ev_connect_result *input,
struct sp_typed_effect_list *effects) {
if (input->outcome != SP_CONNECT_SUCCEEDED && input->outcome != SP_CONNECT_FAILED)
return SP_TCP_CORE_BAD_EVENT;
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_connect_result(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_stream_send_result(struct sp_tcp_core *core,
const struct sp_ev_stream_send_result *input,
struct sp_typed_effect_list *effects) {
if (input->outcome < SP_SEND_PROGRESS || input->outcome > SP_SEND_FAILED)
return SP_TCP_CORE_BAD_EVENT;
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_stream_send_result(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_stream_consumed(struct sp_tcp_core *core,
const struct sp_ev_stream_consumed *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_stream_consumed(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_stream_eof(struct sp_tcp_core *core,
const struct sp_ev_stream_eof *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_stream_eof(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_tun_emit_result(struct sp_tcp_core *core,
const struct sp_ev_tun_emit_result *input,
struct sp_typed_effect_list *effects) {
if (input->outcome < SP_EMIT_ACCEPTED || input->outcome > SP_EMIT_FAILED)
return SP_TCP_CORE_BAD_EVENT;
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_tun_emit_result(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_credit(struct sp_tcp_core *core, const struct sp_ev_credit *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_credit(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_timer_fired(struct sp_tcp_core *core,
const struct sp_ev_timer_fired *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
result = sp_timer(core, input, effects);
return sp_core_transaction_finish(core, effects, &transaction, result);
}
}
enum sp_tcp_core_result
sp_tcp_core_ingress_disposed(struct sp_tcp_core *core,
const struct sp_ev_ingress_disposed *input,
struct sp_typed_effect_list *effects) {
{
struct sp_core_transaction transaction;
enum sp_tcp_core_result result;
int apply;
result = sp_core_transaction_begin(core, &input->context, effects,
&transaction, &apply);
if (!apply)
return result;
if (sp_effect_matches(core, input->context.key, input->effect,
&core->ingress_dispose_effect))
memset(&core->ingress_dispose_effect, 0,
sizeof(core->ingress_dispose_effect));
return sp_core_transaction_finish(core, effects, &transaction,
SP_TCP_CORE_OK);
}
}
#ifndef SOCKPUPPET_TCP_CORE_ONLY
#ifndef AF_ALG
#define AF_ALG 38
#endif
#define MAX_TCP 128
#define MAX_UDP 64
#define MAX_PUBLISH_RULES 32
#define MAX_PUBLISH_UDP 128
#define MAX_EVENTS 64
#define EPOLL_TIMEOUT_MS 100
/* After the broker reaps its child, payload output still in flight gets this
* long (CLOCK_MONOTONIC) to reach its destinations; then the remaining
* relays are abandoned. */
#define SP_CHILD_EXIT_DRAIN_NS (5ULL * 1000000000ULL)
#define TCP_PENDING_WRITE_CAP 262144
#define SP_INGRESS_RESERVATION_SLOT_BYTES TCP_PENDING_WRITE_CAP
#define SP_INGRESS_RESERVATION_SLOTS 16
#define SP_INGRESS_PUBLISH_RESERVATION_SLOTS 8
/* Overridable only to let the retention regression test shrink the FIFO
* below a single flow's flight-window frame count (which the production size of
* 32 exceeds, making the FIFO-full path otherwise unreachable by one flow). */
#ifndef TUN_QUEUE_PACKETS
#define TUN_QUEUE_PACKETS 32
#endif
#define TUN_QUEUE_PACKET_MAX SP_TUN_FRAME_MAX_BYTES
#define RELAY_QUEUE_CAP 262144
#define RELAY_DRAIN_BUDGET 65536
#define RELAY_BATCH_BUDGET 262144
#define PUBLISH_SYNTH_PORT_MIN 32768U
#define PUBLISH_SYNTH_PORT_MAX 60999U
#define SP_BROKER_MEMORY_MAX_BYTES ((size_t)402653184)
#define SP_RESOURCE_MEMORY_MARGIN_BYTES ((size_t)(32 * 1024 * 1024))
#define RLIMIT_PARENT_NOFILE 4096
#define RLIMIT_CHILD_NOFILE 1024
/* Resource geometry is frozen before packet representation and TCP state
* transitions are selected. Configuration may select smaller active limits,
* but these compile-time ceilings remain authority and allocation boundaries. */
enum sp_resource_limit {
SP_VNET_HDR_BYTES = 10,
SP_TUN_BODY_MAX_BYTES = 65535,
SP_TUN_FRAME_MAX_BYTES = SP_VNET_HDR_BYTES + SP_TUN_BODY_MAX_BYTES,
SP_IPV4_HEADER_BYTES = 20,
SP_UDP_HEADER_BYTES = 8,
SP_UDP_PAYLOAD_MAX = 1500 - SP_IPV4_HEADER_BYTES - SP_UDP_HEADER_BYTES,
SP_TCP_MIN_PEER_MSS = 48,
SP_TCP_DEFAULT_PEER_MSS = 536,
SP_TCP_MTU1500_MAX_MSS = 1460,
SP_TCP_GSO_POLICY_SEGS = 32,
SP_TCP_PENDING_PER_FLOW = TCP_PENDING_WRITE_CAP,
SP_TCP_PENDING_ALL =
SP_INGRESS_RESERVATION_SLOTS * SP_INGRESS_RESERVATION_SLOT_BYTES,
SP_TCP_FLIGHT_PER_FLOW = 256 * 1024,
SP_TCP_FLIGHT_ALL = 32 * 1024 * 1024,
SP_TCP_RCV_WSCALE = 3,
SP_UDP_MMSG_MAX = 32,
SP_UDP_PENDING_PER_FLOW = 4,
SP_UDP_PENDING_ALL = 512,
SP_TUN_DATA_MAX_PKTS = 32,
SP_TUN_DATA_MAX_BYTES = 2 * 1024 * 1024,
SP_TUN_CONTROL_MAX_PKTS = 256,
SP_TUN_CONTROL_MAX_BYTES = 512 * 1024,
SP_TUN_CONTROL_FRAME_MAX = SP_VNET_HDR_BYTES + 1500,
SP_TUN_TERMINAL_RESERVE = 16,
SP_TUN_RESPONSE_RESERVE = 8,
SP_INGRESS_STAGE_SLOTS = 12,
SP_EFFECT_BYTES = 256 * 1024,
SP_CHALLENGE_FLOW_PER_S = 10,
SP_CHALLENGE_FLOW_BURST = 20,
SP_CHALLENGE_ALL_PER_S = 100,
SP_CHALLENGE_ALL_BURST = 200,
SP_CLOSED_RST_ALL_PER_S = 100,
SP_CLOSED_RST_ALL_BURST = 200,
SP_ICMP_FLOW_PER_S = 5,
SP_ICMP_FLOW_BURST = 10,
SP_ICMP_ALL_PER_S = 50,
SP_ICMP_ALL_BURST = 100,
SP_TCP_DIRECT_MAX = 96,
SP_TCP_SOCKS_MAX = 64,
SP_TCP_OUTBOUND_MAX = MAX_TCP,
SP_TCP_PUBLISH_MAX = 96,
/* Direct, SOCKS, and published TCP share g_ns_flows. Role ceilings are
* admission policy within this one live-table ceiling, not additive pools. */
SP_TCP_LIVE_ALL_MAX = MAX_TCP,
SP_TCP_HALF_DIRECT_MAX = 32,
SP_TCP_HALF_SOCKS_MAX = 24,
SP_TCP_HALF_OUT_MAX = 48,
SP_TCP_HALF_PUBLISH_MAX = 32,
SP_CONNECT_PUBLISH_MAX = 16,
SP_CONNECT_INFLIGHT_MAX = 32,
SP_TIMEWAIT_OUTBOUND_MAX = 12288,
SP_TIMEWAIT_PUBLISH_MAX = 3072,
SP_TIMEWAIT_ALL_MAX = 15360,
SP_UDP_DIRECT_MAX = 48,
SP_UDP_SOCKS_MAX = 32,
SP_UDP_OUTBOUND_MAX = MAX_UDP,
SP_UDP_PUBLISH_MAX = MAX_PUBLISH_UDP,
SP_UDP_LIVE_ALL_MAX = MAX_UDP + MAX_PUBLISH_UDP,
SP_TCP_TABLE_SLOTS = 256,
SP_UDP_KEY_MAX = MAX_UDP + 2 * MAX_PUBLISH_UDP,
SP_UDP_TABLE_SLOTS = 1024,
SP_TIMEWAIT_TABLE_SLOTS = 32768,
SP_ADMISSION_KEY_MAX = 512,
SP_ADMISSION_TABLE_SLOTS = 1024,
SP_PORT_ALLOC_PROBES = 64,
SP_SYNTH_PORT_FIRST = 32768,
SP_SYNTH_PORT_LAST = 60999,
SP_UDP_ADMIT_ALL_PER_S = 128,
SP_UDP_ADMIT_ALL_BURST = 256,
SP_UDP_ADMIT_SOURCE_S = 32,
SP_UDP_ADMIT_SOURCE_B = 64,
SP_UDP_ADMIT_RULE_S = 64,
SP_UDP_ADMIT_RULE_B = 128,
SP_UDP_SOCKS_HANDSHAKE_SECONDS = 10,
SP_UDP_IDLE_SECONDS = 30,
SP_HOST_TCP_SNDBUF = 128 * 1024,
SP_HOST_TCP_RCVBUF = 128 * 1024,
/* Accepted effective-buffer band. SO_SNDBUF/SO_RCVBUF have no
* deterministic effective value: Linux doubles the request for bookkeeping
* and then clamps to net.core.wmem_max/rmem_max. A host with the common
* 212992 default therefore reports neither the request nor twice it, and a
* tuned-down host legitimately reports less than the request. This band is
* deliberately loose: it catches a set that was ignored outright or a
* nonsensical readback, not lawful kernel clamping. Do not tighten it toward
* SP_HOST_TCP_*BUF without evidence from the hosts in the release lane. */
SP_HOST_TCP_BUF_MIN = 4 * 1024,
SP_HOST_TCP_BUF_MAX = 32 * 1024 * 1024,
SP_HOST_UDP_SNDBUF = 64 * 1024,
SP_HOST_UDP_RCVBUF = 64 * 1024,
SP_CALLBACK_BUDGET_NS = 250 * 1000,
SP_SCHED_ROUND_NS = 2 * 1000 * 1000,
/* These two limits charge expanded logical work in the TUN egress
* service. Keep "egress" in the name: publish ingress has independent
* syscall/item/byte limits below, and calling these generic dispatch limits
* previously hid that the publish callbacks had no bound at all. */
SP_TUN_EGRESS_PKTS_TURN = 64,
SP_TUN_EGRESS_BYTES_TURN = 2 * 1024 * 1024,
SP_PUBLISH_ACCEPTS_TURN = 64,
SP_PUBLISH_DATAGRAMS_TURN = 64,
SP_PUBLISH_BYTES_TURN = 2 * 1024 * 1024,
SP_EFFECTS_TURN = 64,
SP_TIMER_EVENTS_TURN = 64,
/* Descriptors the broker holds while the event loop serves flows, in every
* mode (host tree, rootfs, setup, interactive). Startup-only descriptors
* are closed before the loop and the kernel reuses the lowest free number,
* so they do not add to the peak. test_tun_closer_contract checks a live
* broker against this total. */
SP_FIXED_STDIO_FDS = 3,
SP_FIXED_TUN_FDS = 1,
SP_FIXED_EPOLL_FDS = 1,
/* PID 1's pidfd, its exit-report socket and the liveness pipe write end. */
SP_FIXED_PID1_FDS = 3,
SP_FIXED_TUN_CLOSER_FDS = 1,
/* The stdout/stderr relay pipes plus, when the broker's stdout/stderr is
* not a regular file, a nonblocking destination descriptor for each
* (relay_prepare_destination); or the host tty and PTY master. */
SP_FIXED_PAYLOAD_IO_FDS = 4,
SP_FIXED_SCRATCH_PARENT_FDS = 1,
SP_FIXED_BENCH_TRACE_FDS = 1,
SP_FIXED_AUTHORITY_FDS = SP_FIXED_STDIO_FDS + SP_FIXED_TUN_FDS +
SP_FIXED_EPOLL_FDS + SP_FIXED_PID1_FDS +
SP_FIXED_TUN_CLOSER_FDS + SP_FIXED_PAYLOAD_IO_FDS +
SP_FIXED_SCRATCH_PARENT_FDS +
SP_FIXED_BENCH_TRACE_FDS,
};
#define SP_UDP_PENDING_BYTES \
((size_t)SP_UDP_PENDING_ALL * (size_t)SP_UDP_PAYLOAD_MAX)
#define SP_INGRESS_STAGE_BYTES \
((size_t)SP_INGRESS_STAGE_SLOTS * (size_t)SP_TUN_FRAME_MAX_BYTES)
_Static_assert(SP_VNET_HDR_BYTES == 10,
"SP-07 uses the non-mergeable VNET header");
_Static_assert(SP_ACCEPTED_PROFILE_VERSION == 1,
"accepted profile changes require explicit review");
_Static_assert(SP_TCP_OUTBOUND_MAX <= SP_TCP_LIVE_ALL_MAX &&
SP_TCP_PUBLISH_MAX <= SP_TCP_LIVE_ALL_MAX,
"each TCP role ceiling must fit the shared live table");
_Static_assert(SP_UDP_LIVE_ALL_MAX ==
SP_UDP_OUTBOUND_MAX + SP_UDP_PUBLISH_MAX,
"UDP role ceilings must cover the global ceiling");
_Static_assert(SP_INGRESS_RESERVATION_SLOT_BYTES >=
SP_TCP_GSO_POLICY_SEGS * SP_TCP_MTU1500_MAX_MSS,
"one reservation slot must retain a maximum ingress effect");
_Static_assert(SP_INGRESS_RESERVATION_SLOTS >= SP_INGRESS_STAGE_SLOTS,
"reservation pool must preserve multi-flow progress");
_Static_assert(SP_INGRESS_PUBLISH_RESERVATION_SLOTS <
SP_INGRESS_RESERVATION_SLOTS,
"published peers must not consume every reservation slot");
_Static_assert(SP_TCP_PUBLISH_MAX < SP_TCP_LIVE_ALL_MAX,
"published peers must not consume every default TCP slot");
_Static_assert(SP_TCP_DIRECT_MAX <= SP_TCP_OUTBOUND_MAX &&
SP_TCP_SOCKS_MAX <= SP_TCP_OUTBOUND_MAX,
"outbound TCP role ceilings exceed their table");
_Static_assert(SP_UDP_DIRECT_MAX <= SP_UDP_OUTBOUND_MAX &&
SP_UDP_SOCKS_MAX <= SP_UDP_OUTBOUND_MAX,
"outbound UDP role ceilings exceed their table");
_Static_assert(SP_TCP_LIVE_ALL_MAX * 2 <= SP_TCP_TABLE_SLOTS,
"TCP table load exceeds one half");
_Static_assert(SP_UDP_KEY_MAX * 2 <= SP_UDP_TABLE_SLOTS,
"UDP table load exceeds one half");
_Static_assert(SP_TIMEWAIT_ALL_MAX * 2 <= SP_TIMEWAIT_TABLE_SLOTS,
"TIME-WAIT table load exceeds one half");
_Static_assert(SP_TIMEWAIT_ALL_MAX ==
SP_TIMEWAIT_OUTBOUND_MAX + SP_TIMEWAIT_PUBLISH_MAX,
"TIME-WAIT role ceilings do not cover the global ceiling");
_Static_assert(SP_ADMISSION_KEY_MAX * 2 <= SP_ADMISSION_TABLE_SLOTS,
"admission table load exceeds one half");
_Static_assert(SP_SYNTH_PORT_FIRST > 0 &&
SP_SYNTH_PORT_FIRST <= SP_SYNTH_PORT_LAST,
"synthetic port range is invalid");
_Static_assert(SP_FIXED_AUTHORITY_FDS == 15,
"fixed authority fd worksheet changed");
_Static_assert(SP_UDP_PAYLOAD_MAX == 1472,
"MTU-1500 IHL-5 UDP payload contract");
_Static_assert(SP_TCP_MTU1500_MAX_MSS == 1460,
"MTU-1500 IHL-5 TCP MSS contract");
_Static_assert((int)SP_TCP_CORE_MIN_MSS == (int)SP_TCP_MIN_PEER_MSS &&
(int)SP_TCP_CORE_DEFAULT_MSS ==
(int)SP_TCP_DEFAULT_PEER_MSS &&
(int)SP_TCP_CORE_MAX_MSS ==
(int)SP_TCP_MTU1500_MAX_MSS &&
(int)SP_TCP_CORE_RCV_WSCALE == (int)SP_TCP_RCV_WSCALE &&
(int)SP_TCP_CORE_PENDING_CAPACITY ==
(int)SP_TCP_PENDING_PER_FLOW,
"core limits must remain tied to resource geometry");
_Static_assert(SP_TCP_PENDING_PER_FLOW >= (1U << SP_TCP_RCV_WSCALE),
"receive capacity must cover one scale quantum");
_Static_assert((SP_TCP_PENDING_PER_FLOW >> SP_TCP_RCV_WSCALE) <= UINT16_MAX,
"scaled receive capacity must fit the wire field");
_Static_assert(SP_TCP_PENDING_PER_FLOW -
((SP_TCP_PENDING_PER_FLOW >> SP_TCP_RCV_WSCALE)
<< SP_TCP_RCV_WSCALE) <
(1U << SP_TCP_RCV_WSCALE),
"receive-window floor tail must stay below one quantum");
_Static_assert(SP_TUN_TERMINAL_RESERVE + SP_TUN_RESPONSE_RESERVE <=
SP_TUN_CONTROL_MAX_PKTS,
"control reserves exceed the control queue");
_Static_assert(SP_TUN_CONTROL_MAX_BYTES >=
SP_TUN_CONTROL_MAX_PKTS * SP_TUN_CONTROL_FRAME_MAX,
"control bytes do not cover every control slot");
/* Epoll registrations carry immutable value tokens rather than pointers into
* reusable flow slots. A queued event is accepted only if its type, slot,
* generation, and registered fd still match current state. */
enum fd_type {
FD_TUN = 1,
FD_STDOUT_RELAY,
FD_STDERR_RELAY,
FD_STDOUT_DEST,
FD_STDERR_DEST,
FD_INTERACTIVE_TTY,
FD_INTERACTIVE_PTY,
FD_UDP_RELAY,
FD_UDP_CTRL,
FD_PUBLISH_TCP_LISTENER,
FD_PUBLISH_UDP_SOCKET,
/* Live core-driven TCP adapter backend socket. */
FD_TCP_CORE,
/* Namespace child exit (pidfd). */
FD_CHILD_EXIT
};
struct epoll_key {
enum fd_type type;
uint8_t index;
uint16_t fd;
uint32_t generation;
};
static uint64_t epoll_token_encode(enum fd_type type, uint8_t index,
uint16_t fd, uint32_t generation) {
return ((uint64_t)(uint8_t)type << 56) | ((uint64_t)index << 48) |
((uint64_t)fd << 32) | (uint64_t)generation;
}
static int epoll_token_decode(uint64_t token, struct epoll_key *key) {
uint8_t type = (uint8_t)(token >> 56);
if (key == NULL || type < (uint8_t)FD_TUN ||
type > (uint8_t)FD_CHILD_EXIT)
return -1;
key->type = (enum fd_type)type;
key->index = (uint8_t)(token >> 48);
key->fd = (uint16_t)(token >> 32);
key->generation = (uint32_t)token;
if (key->generation == 0)
return -1;
return 0;
}
static uint32_t epoll_next_generation(uint32_t generation) {
generation++;
return generation == 0 ? 1 : generation;
}
static int epoll_key_matches(const struct epoll_key *key, enum fd_type type,
uint8_t index, int fd, uint32_t generation) {
return key != NULL && fd >= 0 && fd <= UINT16_MAX && generation != 0 &&
key->type == type && key->index == index &&
key->fd == (uint16_t)fd && key->generation == generation;
}
static int g_epfd = -1;
static int g_tun_registered_fd = -1;
static struct epoll_event g_tun_interest;
/* Tokens include fd, role, slot and generation; cache only installed state. */
static int sp_epoll_update(int operation, int fd, struct epoll_event *event,
struct epoll_event *installed) {
if (operation == EPOLL_CTL_MOD && installed->events == event->events &&
installed->data.u64 == event->data.u64)
return 0;
if (epoll_ctl(g_epfd, operation, fd, event) < 0)
return -1;
*installed = *event;
return 0;
}
/* The relay state is deliberately Sans-I/O: it owns bytes and transitions,
* but no descriptors, errno values, clocks, or syscalls. The broker runtime
* below translates readiness and syscall results into these operations. */
struct sp_relay_state {
uint8_t *storage;
size_t capacity;
size_t head;
size_t length;
int source_eof;
int destination_failed;
int source_paused;
uint64_t bytes_out;
};
static uint8_t g_stdout_relay_storage[RELAY_QUEUE_CAP];
static uint8_t g_stderr_relay_storage[RELAY_QUEUE_CAP];
static uint8_t g_tty_to_pty_storage[RELAY_QUEUE_CAP];
static uint8_t g_pty_to_tty_storage[RELAY_QUEUE_CAP];
static void sp_relay_init(struct sp_relay_state *relay, uint8_t *storage,
size_t capacity) {
memset(relay, 0, sizeof(*relay));
relay->storage = storage;
relay->capacity = capacity;
}
static size_t sp_relay_space(const struct sp_relay_state *relay) {
return relay->capacity - relay->length;
}
static size_t sp_relay_accept(struct sp_relay_state *relay,
const uint8_t *data, size_t length) {
size_t accepted = length < sp_relay_space(relay) ? length
: sp_relay_space(relay);
size_t tail = (relay->head + relay->length) % relay->capacity;
size_t first = accepted < relay->capacity - tail ? accepted
: relay->capacity - tail;
if (first > 0)
memcpy(relay->storage + tail, data, first);
if (accepted > first)
memcpy(relay->storage, data + first, accepted - first);
relay->length += accepted;
if (relay->length == relay->capacity)
relay->source_paused = 1;
return accepted;
}
static const uint8_t *sp_relay_output(const struct sp_relay_state *relay,
size_t *length) {
size_t contiguous = relay->length;
if (contiguous > relay->capacity - relay->head)
contiguous = relay->capacity - relay->head;
*length = contiguous;
return relay->storage + relay->head;
}
static void sp_relay_consume(struct sp_relay_state *relay, size_t length) {
if (length > relay->length)
length = relay->length;
relay->head = (relay->head + length) % relay->capacity;
relay->length -= length;
relay->bytes_out += length;
if (relay->source_paused && relay->length <= relay->capacity / 2)
relay->source_paused = 0;
}
static void sp_relay_source_eof(struct sp_relay_state *relay) {
relay->source_eof = 1;
relay->source_paused = 0;
}
static void sp_relay_destination_failed(struct sp_relay_state *relay) {
relay->destination_failed = 1;
relay->head = 0;
relay->length = 0;
relay->source_paused = 0;
}
static int sp_relay_wants_read(const struct sp_relay_state *relay) {
return !relay->source_eof && !relay->destination_failed &&
!relay->source_paused && relay->length < relay->capacity;
}
static int sp_relay_wants_write(const struct sp_relay_state *relay) {
return !relay->destination_failed && relay->length > 0;
}
static int sp_relay_complete(const struct sp_relay_state *relay) {
return relay->destination_failed || (relay->source_eof && relay->length == 0);
}
enum sp_relay_event_type {
SP_RELAY_SOURCE_BYTES = 1,
SP_RELAY_SOURCE_EOF,
SP_RELAY_DESTINATION_WRITTEN,
SP_RELAY_DESTINATION_FAILED
};
struct sp_relay_event {
enum sp_relay_event_type type;
const uint8_t *bytes;
size_t length;
};
enum sp_relay_action {
SP_RELAY_ACTION_READ_SOURCE = 1U << 0,
SP_RELAY_ACTION_WRITE_DESTINATION = 1U << 1,
SP_RELAY_ACTION_COMPLETE = 1U << 2,
SP_RELAY_ACTION_INVALID = 1U << 3
};
static unsigned int sp_relay_step(struct sp_relay_state *relay,
const struct sp_relay_event *event) {
switch (event->type) {
case SP_RELAY_SOURCE_BYTES:
if ((event->length > 0 && event->bytes == NULL) ||
event->length > sp_relay_space(relay))
return SP_RELAY_ACTION_INVALID;
(void)sp_relay_accept(relay, event->bytes, event->length);
break;
case SP_RELAY_SOURCE_EOF:
sp_relay_source_eof(relay);
break;
case SP_RELAY_DESTINATION_WRITTEN:
if (event->length > relay->length)
return SP_RELAY_ACTION_INVALID;
sp_relay_consume(relay, event->length);
break;
case SP_RELAY_DESTINATION_FAILED:
sp_relay_destination_failed(relay);
break;
default:
return SP_RELAY_ACTION_INVALID;
}
unsigned int actions = 0;
if (sp_relay_wants_read(relay))
actions |= SP_RELAY_ACTION_READ_SOURCE;
if (sp_relay_wants_write(relay))
actions |= SP_RELAY_ACTION_WRITE_DESTINATION;
if (sp_relay_complete(relay))
actions |= SP_RELAY_ACTION_COMPLETE;
return actions;
}
/* The kernel TUN write queue is finite. Keep a small, fixed broker-side
* queue so a non-reading child cannot block the event loop indefinitely.
* Packets are never reordered: once there is queued output, later packets
* join the queue even if the fd becomes writable before the next EPOLLOUT. */
struct tun_packet {
size_t len;
uint8_t data[TUN_QUEUE_PACKET_MAX];
};
struct tun_queue {
struct tun_packet packets[TUN_QUEUE_PACKETS];
size_t head;
size_t count;
};
static struct tun_queue g_tun_queue;
/* One authoritative schema keeps the runtime counter names, storage, and
* report enumeration synchronized. Each entry names one stable scalar
* uint64_t field; update sites may use saturating increments or direct gauge
* assignment. Array histograms and fixed history records remain separate
* evidence products because they have different output contracts. */
#define SP_RESOURCE_COUNTER_SCHEMA(X) \
X(tun_queue_drops) \
X(tun_vnet_scalar_egress) \
X(host_datagram_read_calls) \
X(host_datagram_read_bytes) \
X(host_datagram_write_bytes) \
X(tcp_logical_segments_host_to_child) \
X(tcp_gso_frames_child_to_host) \
X(tcp_gso_expand_rejects) \
X(tcp_gso_frames_host_to_child) \
X(tcp_gso_segments_host_to_child) \
X(tcp_payload_bytes_host_to_child) \
X(parser_rejects) \
X(pump_reentry_failures) \
X(reservation_current) \
X(reservation_high_water) \
X(reservation_acquire_failures) \
X(tcp_flow_rejections) \
X(relay_drain_abandoned) \
X(epoll_update_failures) \
X(udp_flow_rejections) \
X(udp_queue_slot_exhaustions) \
X(udp_queue_byte_exhaustions) \
X(udp_terminal_drops) \
X(udp_recvmmsg_calls) \
X(udp_recvmmsg_datagrams) \
X(udp_recvmmsg_zero_results) \
X(udp_recvmmsg_eintr_yields) \
X(udp_sendmmsg_datagrams) \
X(udp_sendmmsg_partial_prefixes) \
X(udp_sendmmsg_zero_results) \
X(udp_sendmmsg_eintr_yields) \
X(ipv4_fragment_drops) \
X(scheduler_stale_index_rejects) \
X(icmp_drops)
#define SP_RESOURCE_COUNTER_FIELD(name) uint64_t name;
struct sp_resource_counters {
SP_RESOURCE_COUNTER_SCHEMA(SP_RESOURCE_COUNTER_FIELD)
uint64_t tun_queue_high_water;
};
#undef SP_RESOURCE_COUNTER_FIELD
static struct sp_resource_counters g_resource_counters;
static void sp_resource_print_baseline(const char *point);
static int g_ns_sched_tun_pending;
static int g_resource_metrics_enabled;
static pid_t g_runtime_role_pid;
static int g_vnet_profile_validated;
/* Counters and gauges are maintained only when SOCKPUPPET_METRICS asked for
* the exit dump; otherwise every update is skipped. No production decision
* reads them. */
static void sp_counter_add(uint64_t *counter, uint64_t amount) {
if (!g_resource_metrics_enabled)
return;
if (UINT64_MAX - *counter < amount)
*counter = UINT64_MAX;
else
*counter += amount;
}
static void sp_counter_inc(uint64_t *counter) {
sp_counter_add(counter, 1);
}
static void sp_resource_count_host_io(int datagram, int write_direction,
ssize_t result) {
if (!g_resource_metrics_enabled || !datagram)
return;
if (!write_direction)
sp_counter_inc(&g_resource_counters.host_datagram_read_calls);
if (result > 0)
sp_counter_add(write_direction
? &g_resource_counters.host_datagram_write_bytes
: &g_resource_counters.host_datagram_read_bytes,
(uint64_t)result);
}
struct socks_config {
char host[256];
int port;
char username[256];
char password[256];
int enabled;
int remote_dns;
int addr_valid;
struct sockaddr_in addr;
};
enum socks_io_state {
SOCKS_IO_NONE = 0,
SOCKS_IO_CONNECTING,
SOCKS_IO_METHOD,
SOCKS_IO_AUTH,
SOCKS_IO_REQUEST,
SOCKS_IO_READY,
SOCKS_IO_FAILED
};
struct socks_io {
int is_udp;
int connect_pending;
enum socks_io_state state;
uint32_t target_ip;
uint16_t target_port;
char target_domain[256];
uint8_t txbuf[512];
size_t tx_off;
size_t tx_len;
uint8_t rxbuf[512];
size_t rx_len;
};
static struct socks_config socks_proxy = {0};
static int require_writable_cwd = 0;
static int interactive_stdio = 0;
static int verbose = 0;
static int tcp_flow_limit = MAX_TCP;
static int udp_flow_limit = MAX_UDP;
#define SP_BIND_NAME_MAX 32
#define SP_SETUP_SCRIPT_MAX (1024U * 1024U)
#define SP_SETUP_TIMEOUT_DEFAULT_MS 30000U
#define SP_SETUP_TIMEOUT_MAX_MS 300000U
enum sp_bind_mode {
SP_BIND_RO,
SP_BIND_RW,
SP_BIND_COW,
};
enum sp_view_mode {
SP_VIEW_READONLY,
SP_VIEW_COW_DISPOSABLE,
SP_VIEW_COW_RETAINED,
};
enum sp_scope_kind {
SP_SCOPE_ROOTFS,
SP_SCOPE_WORKSPACE,
SP_SCOPE_BIND,
};
enum sp_upper_mode {
SP_UPPER_NONE,
SP_UPPER_DISPOSABLE,
SP_UPPER_RETAINED,
};
struct sp_bind_spec {
char name[SP_BIND_NAME_MAX + 1];
const char *source;
const char *target;
enum sp_bind_mode mode;
};
struct sp_fs_config {
const char *rootfs_path;
const char *retained_directory;
enum sp_view_mode rootfs_view;
enum sp_view_mode workspace_view;
struct sp_bind_spec bind;
int bind_enabled;
};
struct sp_setup_config {
const char *host_script;
const char *guest_script;
unsigned int timeout_ms;
uid_t payload_uid;
gid_t payload_gid;
};
struct sp_launch_config {
struct sp_fs_config filesystem;
struct sp_setup_config setup;
int setup_enabled;
};
struct sp_prepared_fs {
int retained_lock_fd;
int scratch_parent_fd;
char scratch_name[64];
int scratch_owned;
};
struct sp_mount_scope {
const struct sp_bind_spec *bind;
enum sp_scope_kind kind;
enum sp_bind_mode mode;
enum sp_upper_mode upper_mode;
int source_fd;
int target_fd;
int upper_fd;
int work_fd;
int attached;
};
/* The CLI-selected TCP limit is the shared direct/SOCKS/publish live-table
* boundary. Startup rejects values outside this range; returning zero here
* also makes focused tests and any future accidental pre-validation use fail
* closed instead of silently falling back to MAX_TCP. */
static size_t sp_ns_tcp_slot_limit(void) {
return tcp_flow_limit >= 1 && tcp_flow_limit <= MAX_TCP
? (size_t)tcp_flow_limit
: 0;
}
static size_t sp_publish_limit_for_shared(size_t shared) {
size_t child_reserved;
size_t publish;
if (shared == 0)
return 0;
child_reserved = shared / 4 + (shared % 4 != 0);
publish = shared - child_reserved;
return publish < SP_TCP_PUBLISH_MAX ? publish : SP_TCP_PUBLISH_MAX;
}
static volatile sig_atomic_t interactive_resize_pending = 0;
struct interactive_session {
int active;
int host_tty_fd;
int pty_master_fd;
int pty_slave_fd;
struct termios host_termios;
struct winsize host_winsize;
int host_termios_saved;
};
static struct interactive_session interactive_session = {
.host_tty_fd = -1,
.pty_master_fd = -1,
.pty_slave_fd = -1,
};
/*
* Keep post-seccomp diagnostics deterministic on write(2). The parent also
* admits readv(2)/writev(2) for libc and ordinary vectored-I/O compatibility,
* but this fixed buffer avoids depending on a particular stdio backend or
* introducing allocation into the hardened path. stderr is an intentionally
* synchronous, potentially blocking sink; steady-state packet diagnostics
* must therefore use counters or rate-limited samples.
* Submit each record in one write (retry only EINTR with no bytes written).
* A short write is an error, not a second fragment that could interleave.
*/
/* Set while stderr is the host terminal in --interactive raw mode, where a
* bare newline would not return the cursor to column 0. Forked sandbox
* processes inherit it. */
static int g_stderr_raw_tty;
__attribute__((format(printf, 1, 2))) static int
sp_stderr_printf(const char *fmt, ...) {
char buffer[2048];
char crlf[2 * sizeof(buffer)];
const char *output = buffer;
va_list ap;
int length;
ssize_t written;
_Static_assert(sizeof(buffer) <= PIPE_BUF, "stderr record must fit a pipe write");
va_start(ap, fmt);
length = vsnprintf(buffer, sizeof(buffer), fmt, ap);
va_end(ap);
if (length < 0)
return -1;
if ((size_t)length >= sizeof(buffer)) {
errno = EOVERFLOW;
return -1;
}
if (g_stderr_raw_tty) {
int expanded = 0;
for (int i = 0; i < length; ++i) {
if (buffer[i] == '\n')
crlf[expanded++] = '\r';
crlf[expanded++] = buffer[i];
}
output = crlf;
length = expanded;
}
do {
written = write(STDERR_FILENO, output, (size_t)length);
} while (written < 0 && errno == EINTR);
if (written < 0)
return -1;
if (written != length) {
errno = EIO;
return -1;
}
return 0;
}
/* Debug macro - only prints if verbose mode enabled. */
#define DBG(fmt, ...) \
do { \
if (verbose) \
(void)sp_stderr_printf("[sockpuppet] " fmt "\n", ##__VA_ARGS__); \
} while (0)
/* Host gateway configuration - map 10.0.1.x to 127.0.0.x */
#define SANDBOX_GATEWAY_IP 0x0100000a /* 10.0.0.1: the sandbox's next hop */
#define DNS_PROXY_IP SANDBOX_GATEWAY_IP /* broker DNS for socks5h */
#define HOST_GATEWAY_BASE 0x0001000a /* 10.0.1.0 network byte order */
#define HOST_GATEWAY_MASK 0x00ffffff /* /24 mask for 10.0.1.x */
#define LOCALHOST_BASE 0x0000007f /* 127.0.0.0 network byte order */
#define SANDBOX_IP 0x0200000a /* 10.0.0.2 network byte order */
#define PUBLISH_SYNTHETIC_IP 0x0102000a /* 10.0.2.1 network byte order */
#define MAX_HOST_RULES 64
#define MAX_DIRECT_RULES 64
#define MAX_DNS_MAPPINGS 1024
#define DNS_SYNTHETIC_BASE 0x0a3f0000U /* 10.63.0.0/16, host byte order */
#define SP_DNS_SLOT_INVALID UINT16_MAX
#define SP_DNS_NAME_MAX 255
#define SP_DNS_QUESTION_MAX (SP_DNS_NAME_MAX + 6)
#define SP_DNS_QUERY_MAX 512
#define SP_DNS_RESPONSE_MAX 512
#define SP_DNS_TTL_SECONDS 60U
#define SP_DNS_TTL_NS \
((uint64_t)SP_DNS_TTL_SECONDS * UINT64_C(1000000000))
enum egress_mode {
EGRESS_DIRECT = 0,
EGRESS_SOCKS,
EGRESS_NONE,
};
enum parent_harden_profile {
PARENT_HARDEN_DIRECT = 0,
PARENT_HARDEN_SOCKS,
PARENT_HARDEN_EGRESS_NONE,
};
struct parent_harden_config {
enum parent_harden_profile profile;
int outbound_sockets_allowed;
int interactive;
/* Fatal-teardown signaling targets, pinned into the parent seccomp
* filter: signaling is allowed only as SIGKILL to exactly these. */
pid_t child_pid;
int child_pidfd;
};
static enum egress_mode egress_mode = EGRESS_DIRECT;
static int egress_mode_explicit = 0;
struct host_rule {
uint8_t last_octet; /* exact x in 127.0.0.x (0-255) */
uint16_t port; /* exact port number (1-65535) */
int proto; /* IPPROTO_TCP, IPPROTO_UDP, or 0 for both */
};
static struct host_rule host_rules[MAX_HOST_RULES];
static int host_rule_count = 0;
struct direct_rule {
uint32_t network; /* host byte order */
uint32_t mask; /* host byte order */
uint16_t port;
int proto; /* IPPROTO_TCP, IPPROTO_UDP, or 0 for both */
};
static struct direct_rule direct_rules[MAX_DIRECT_RULES];
static int direct_rule_count = 0;
struct publish_rule {
uint32_t host_ip; /* network byte order */
uint16_t host_port;
uint16_t container_port;
int proto; /* IPPROTO_TCP or IPPROTO_UDP */
int listen_fd;
int rule_index;
int effective_sndbuf;
int effective_rcvbuf;
struct epoll_event interest;
};
static struct publish_rule publish_rules[MAX_PUBLISH_RULES];
static int publish_rule_count = 0;
/* Descriptor-free, fixed-size policy product sealed after CLI validation.
* Credentials, resolved proxy sockaddr values, listener descriptors, socket
* buffers, and counters are executor state and intentionally absent. */
struct sp_net_policy_snapshot {
uint8_t initialized;
uint8_t egress;
uint8_t socks_remote_dns;
uint16_t host_rule_count;
uint16_t direct_rule_count;
struct host_rule host_rules[MAX_HOST_RULES];
struct direct_rule direct_rules[MAX_DIRECT_RULES];
};
static struct sp_net_policy_snapshot g_net_policy;
static int sp_policy_proto_valid(int proto, int allow_both) {
return proto == IPPROTO_TCP || proto == IPPROTO_UDP ||
(allow_both && proto == 0);
}
static int sp_policy_mask_valid(uint32_t mask) {
uint32_t inverse = ~mask;
return (inverse & (inverse + 1U)) == 0;
}
static int sp_net_policy_build(
struct sp_net_policy_snapshot *out, enum egress_mode selected_egress,
int selected_socks_enabled, int selected_socks_remote_dns,
const struct host_rule *selected_host_rules,
int selected_host_rule_count,
const struct direct_rule *selected_direct_rules,
int selected_direct_rule_count,
const struct publish_rule *selected_publish_rules,
int selected_publish_rule_count) {
struct sp_net_policy_snapshot candidate;
if (out == NULL || selected_egress < EGRESS_DIRECT ||
selected_egress > EGRESS_NONE ||
(selected_socks_enabled != 0 && selected_socks_enabled != 1) ||
(selected_socks_remote_dns != 0 && selected_socks_remote_dns != 1) ||
selected_host_rule_count < 0 ||
selected_host_rule_count > MAX_HOST_RULES ||
selected_direct_rule_count < 0 ||
selected_direct_rule_count > MAX_DIRECT_RULES ||
selected_publish_rule_count < 0 ||
selected_publish_rule_count > MAX_PUBLISH_RULES ||
(selected_host_rule_count != 0 && selected_host_rules == NULL) ||
(selected_direct_rule_count != 0 && selected_direct_rules == NULL) ||
(selected_publish_rule_count != 0 && selected_publish_rules == NULL))
return -1;
if ((selected_egress == EGRESS_SOCKS) != selected_socks_enabled ||
(selected_socks_remote_dns && !selected_socks_enabled))
return -1;
memset(&candidate, 0, sizeof(candidate));
candidate.egress = (uint8_t)selected_egress;
candidate.socks_remote_dns = (uint8_t)selected_socks_remote_dns;
candidate.host_rule_count = (uint16_t)selected_host_rule_count;
candidate.direct_rule_count = (uint16_t)selected_direct_rule_count;
for (int i = 0; i < selected_host_rule_count; ++i) {
const struct host_rule *rule = &selected_host_rules[i];
if (rule->port == 0 || !sp_policy_proto_valid(rule->proto, 1))
return -1;
candidate.host_rules[i] = *rule;
}
for (int i = 0; i < selected_direct_rule_count; ++i) {
const struct direct_rule *rule = &selected_direct_rules[i];
if (rule->port == 0 || !sp_policy_proto_valid(rule->proto, 1) ||
!sp_policy_mask_valid(rule->mask) ||
(rule->network & ~rule->mask) != 0)
return -1;
candidate.direct_rules[i] = *rule;
}
for (int i = 0; i < selected_publish_rule_count; ++i) {
const struct publish_rule *rule = &selected_publish_rules[i];
uint32_t host = ntohl(rule->host_ip);
if ((host & 0xff000000U) != 0x7f000000U || host == 0x7f000000U ||
host == 0x7f000001U || host == 0x7fffffffU ||
rule->host_port == 0 || rule->container_port == 0 ||
!sp_policy_proto_valid(rule->proto, 0) || rule->rule_index != i)
return -1;
}
candidate.initialized = 1;
*out = candidate;
return 0;
}
/* Private fixed-capacity index. SipHash-1-3 filters; adapters compare typed
* keys. Probes never escape a synchronous operation. Only bucket metadata moves
* on erase; object slots and their timer/reference identities remain stable.
* Allocation and entropy belong to broker startup, outside these mechanics. */
#if !defined(__linux__)
#error "hash table requires Linux"
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
#error "hash table requires little-endian"
#endif
#if !defined(__x86_64__) && !defined(__aarch64__) && \
!(defined(__riscv) && __riscv_xlen == 64)
#error "supported targets are x86-64, AArch64 and RV64"
#endif
_Static_assert(CHAR_BIT == 8 && sizeof(void *) == 8,
"64-bit byte-addressed target required");
struct ht_hash_key {
uint64_t k0;
uint64_t k1;
};
static inline uint64_t
ht_rotl64(uint64_t x, unsigned n)
{
return (x << n) | (x >> (64 - n));
}
#define HT_SIPROUND() do { \
v0 += v1; v1 = ht_rotl64(v1, 13); \
v1 ^= v0; v0 = ht_rotl64(v0, 32); \
v2 += v3; v3 = ht_rotl64(v3, 16); \
v3 ^= v2; \
v0 += v3; v3 = ht_rotl64(v3, 21); \
v3 ^= v0; \
v2 += v1; v1 = ht_rotl64(v1, 17); \
v1 ^= v2; v2 = ht_rotl64(v2, 32); \
} while (0)
static inline uint64_t
ht_load64(const void *p)
{
uint64_t x;
memcpy(&x, p, sizeof x);
return x;
}
static inline uint64_t
ht_siphash13(const void *data, size_t len,
const struct ht_hash_key *key)
{
const uint8_t *p = data;
const uint8_t *end = p + (len & ~(size_t)7);
uint64_t v0 = UINT64_C(0x736f6d6570736575) ^ key->k0;
uint64_t v1 = UINT64_C(0x646f72616e646f6d) ^ key->k1;
uint64_t v2 = UINT64_C(0x6c7967656e657261) ^ key->k0;
uint64_t v3 = UINT64_C(0x7465646279746573) ^ key->k1;
uint64_t b = (uint64_t)len << 56;
while (p != end) {
uint64_t m = ht_load64(p);
v3 ^= m;
HT_SIPROUND();
v0 ^= m;
p += 8;
}
switch (len & 7) {
case 7: b |= (uint64_t)p[6] << 48; /* fall through */
case 6: b |= (uint64_t)p[5] << 40; /* fall through */
case 5: b |= (uint64_t)p[4] << 32; /* fall through */
case 4: b |= (uint64_t)p[3] << 24; /* fall through */
case 3: b |= (uint64_t)p[2] << 16; /* fall through */
case 2: b |= (uint64_t)p[1] << 8; /* fall through */
case 1: b |= (uint64_t)p[0]; /* fall through */
default:
break;
}
v3 ^= b;
HT_SIPROUND();
v0 ^= b;
v2 ^= UINT64_C(0xff);
HT_SIPROUND();
HT_SIPROUND();
HT_SIPROUND();
return v0 ^ v1 ^ v2 ^ v3;
}
static inline bool
ht_hash_key_random(struct ht_hash_key *key)
{
uint8_t *p;
size_t left;
if (!key)
return false;
p = (uint8_t *)key;
left = sizeof *key;
while (left) {
ssize_t n = getrandom(p, left, 0);
if (n > 0) {
p += (size_t)n;
left -= (size_t)n;
continue;
}
if (n < 0 && errno == EINTR)
continue;
return false;
}
return true;
}
#undef HT_SIPROUND
#define HT_SLOT_EMPTY UINT32_MAX
#define HT_SLOT_ERROR (UINT32_MAX - 1U)
struct ht_bucket {
uint32_t hash;
uint32_t slot;
};
_Static_assert(sizeof(struct ht_bucket) == 8,
"ht_bucket must stay 8 bytes");
struct ht_index {
struct ht_bucket *b;
uint32_t capacity;
uint32_t slot_count;
uint32_t count;
uint32_t limit;
};
struct ht_probe {
uint32_t pos;
uint32_t left;
uint32_t hash;
};
enum ht_step {
HT_CANDIDATE,
HT_VACANT,
HT_EXHAUSTED,
HT_CORRUPT
};
enum ht_insert_result {
HT_INSERTED,
HT_FULL = 2,
HT_INVALID
};
static inline bool
ht_init(struct ht_index *t,
struct ht_bucket *bucket_storage,
uint32_t capacity,
uint32_t slot_count,
uint32_t limit)
{
if (!t || !bucket_storage ||
capacity < 2 ||
(capacity & (capacity - 1U)) ||
!slot_count ||
slot_count >= HT_SLOT_ERROR ||
!limit ||
limit > capacity / 2U)
return false;
for (uint32_t i = 0; i < capacity; ++i)
bucket_storage[i].slot = HT_SLOT_EMPTY;
*t = (struct ht_index) {
.b = bucket_storage,
.capacity = capacity,
.slot_count = slot_count,
.count = 0,
.limit = limit,
};
return true;
}
static inline struct ht_probe
ht_probe_begin(const struct ht_index *t, uint64_t hash)
{
uint32_t h = (uint32_t)hash;
return (struct ht_probe) {
.pos = h & (t->capacity - 1U),
.left = t->capacity,
.hash = h,
};
}
static inline enum ht_step
ht_probe_next(const struct ht_index *t,
struct ht_probe *p,
uint32_t *slot,
uint32_t *bucket)
{
if (!t || !p || !slot || !bucket || !t->b || !t->capacity)
return HT_CORRUPT;
const uint32_t mask = t->capacity - 1U;
while (p->left) {
--p->left;
const uint32_t i = p->pos;
const struct ht_bucket *b = &t->b[i];
p->pos = (i + 1U) & mask;
*bucket = i;
if (b->slot == HT_SLOT_EMPTY) {
*slot = HT_SLOT_EMPTY;
return HT_VACANT;
}
if (b->slot >= t->slot_count) {
*slot = HT_SLOT_ERROR;
return HT_CORRUPT;
}
if (b->hash == p->hash) {
*slot = b->slot;
return HT_CANDIDATE;
}
}
*slot = HT_SLOT_EMPTY;
*bucket = t->capacity;
return HT_EXHAUSTED;
}
static inline enum ht_insert_result
ht_insert_at(struct ht_index *t,
const struct ht_probe *p,
uint32_t bucket,
uint32_t slot)
{
if (!t || !p || !t->b ||
bucket >= t->capacity ||
slot >= t->slot_count ||
t->b[bucket].slot != HT_SLOT_EMPTY)
return HT_INVALID;
if (t->count >= t->limit)
return HT_FULL;
t->b[bucket] = (struct ht_bucket) {
.hash = p->hash,
.slot = slot,
};
++t->count;
return HT_INSERTED;
}
static inline uint32_t
ht_erase_at(struct ht_index *t, uint32_t hole)
{
if (!t || !t->b ||
hole >= t->capacity ||
!t->count ||
t->b[hole].slot == HT_SLOT_EMPTY ||
t->b[hole].slot >= t->slot_count)
return HT_SLOT_ERROR;
const uint32_t mask = t->capacity - 1U;
uint32_t i = (hole + 1U) & mask;
bool found_empty = false;
for (uint32_t n = 0;
n < t->capacity;
++n, i = (i + 1U) & mask) {
uint32_t slot = t->b[i].slot;
if (slot == HT_SLOT_EMPTY) {
found_empty = true;
break;
}
if (slot >= t->slot_count)
return HT_SLOT_ERROR;
}
if (!found_empty)
return HT_SLOT_ERROR;
const uint32_t removed = t->b[hole].slot;
i = (hole + 1U) & mask;
for (;;) {
struct ht_bucket b = t->b[i];
if (b.slot == HT_SLOT_EMPTY)
break;
const uint32_t home = b.hash & mask;
if (((hole - home) & mask) <
((i - home) & mask)) {
t->b[hole] = b;
hole = i;
}
i = (i + 1U) & mask;
}
t->b[hole].slot = HT_SLOT_EMPTY;
--t->count;
return removed;
}
struct sp_admission_key { uint8_t bytes[5]; };
struct sp_admission_entry;
static void sp_resource_admission_source_key(struct sp_admission_key *key, uint32_t cli_ip,
uint8_t role);
static struct sp_admission_entry *
sp_resource_admission_acquire(const struct sp_admission_key *key);
static int sp_resource_admission_take(struct sp_admission_entry *entry,
uint64_t now_ns);
static void sp_resource_admission_release_source(uint32_t cli_ip, uint8_t role);
static int sp_ns_accept_publish(int tunfd, struct publish_rule *rule, int fd,
const struct sockaddr_in *peer);
enum sp_udp_role {
SP_UDP_ROLE_NONE = 0,
SP_UDP_ROLE_DIRECT,
SP_UDP_ROLE_SOCKS,
SP_UDP_ROLE_PUBLISH,
};
enum sp_udp_mapping_state {
SP_UDP_MAPPING_INACTIVE = 0,
SP_UDP_MAPPING_ALLOCATED,
SP_UDP_MAPPING_SOCKS_NEGOTIATING,
SP_UDP_MAPPING_READY,
SP_UDP_MAPPING_DRAINING,
};
/* One indexed deadline heap. Owner IDs are stable table slots; teardown
* removes the slot before reuse. Position zero means absent, so BSS is ready
* without touching payload arenas or accumulating stale timer entries. */
enum {
SP_DEADLINE_TCP = 0,
SP_DEADLINE_UDP = MAX_TCP,
SP_DEADLINE_PUBLISH = MAX_TCP + MAX_UDP,
SP_DEADLINE_TIMEWAIT = MAX_TCP + MAX_UDP + MAX_PUBLISH_UDP,
SP_DEADLINE_COUNT = SP_DEADLINE_TIMEWAIT + SP_TIMEWAIT_ALL_MAX,
};
struct sp_deadline_heap {
uint64_t deadlines[SP_DEADLINE_COUNT];
uint16_t owners[SP_DEADLINE_COUNT];
uint16_t positions[SP_DEADLINE_COUNT]; /* heap index plus one */
uint16_t count;
};
_Static_assert(SP_DEADLINE_COUNT < UINT16_MAX, "deadline indices fit");
static struct sp_deadline_heap g_net_deadlines;
static int sp_deadline_less(const struct sp_deadline_heap *h, unsigned a, unsigned b) {
uint16_t x = h->owners[a], y = h->owners[b];
return h->deadlines[x] < h->deadlines[y] ||
(h->deadlines[x] == h->deadlines[y] && x < y);
}
static void sp_deadline_swap(struct sp_deadline_heap *h, unsigned a, unsigned b) {
uint16_t x = h->owners[a];
h->owners[a] = h->owners[b];
h->owners[b] = x;
h->positions[h->owners[a]] = (uint16_t)(a + 1);
h->positions[x] = (uint16_t)(b + 1);
}
static void sp_deadline_set(struct sp_deadline_heap *h, unsigned owner, uint64_t deadline) {
unsigned pos = h->positions[owner];
h->deadlines[owner] = deadline;
if (pos == 0) {
if (deadline == 0) return;
pos = h->count++;
h->owners[pos] = (uint16_t)owner;
h->positions[owner] = (uint16_t)(pos + 1);
} else {
--pos;
if (deadline == 0) {
h->positions[owner] = 0;
--h->count;
if (pos == h->count) return;
h->owners[pos] = h->owners[h->count];
h->positions[h->owners[pos]] = (uint16_t)(pos + 1);
}
}
while (pos > 0 && sp_deadline_less(h, pos, (pos - 1) / 2)) {
unsigned parent = (pos - 1) / 2;
sp_deadline_swap(h, pos, parent);
pos = parent;
}
for (;;) {
unsigned child = 2 * pos + 1;
if (child >= h->count) break;
if (child + 1 < h->count && sp_deadline_less(h, child + 1, child)) ++child;
if (!sp_deadline_less(h, child, pos)) break;
sp_deadline_swap(h, pos, child);
pos = child;
}
}
static uint64_t sp_deadline_next(const struct sp_deadline_heap *h) {
return h->count ? h->deadlines[h->owners[0]] : 0;
}
/* Sans-I/O UDP: shared lifecycle and datagram-handle queue for all roles.
* Payload arenas, tuple lookup, descriptors and admission accounting are adapter-owned. */
struct sp_udp_core {
uint64_t idle_deadline_ns;
uint64_t handshake_deadline_ns;
uint16_t pending_slots[SP_UDP_PENDING_PER_FLOW];
uint8_t role;
uint8_t state;
uint8_t pending_head;
uint8_t pending_count;
};
enum sp_udp_event { SP_UDP_BEGIN, SP_UDP_NEGOTIATE, SP_UDP_READY,
SP_UDP_PROGRESS, SP_UDP_EXPIRE, SP_UDP_CLOSE };
static uint64_t sp_udp_deadline(uint64_t now, unsigned seconds) {
uint64_t duration = (uint64_t)seconds * UINT64_C(1000000000);
return UINT64_MAX - now < duration ? UINT64_MAX : now + duration;
}
/* Returns one close command on expiry; the adapter releases sockets and slots.
* Queue credit is independent of state, so a blocked send never refreshes idle time. */
static int sp_udp_step(struct sp_udp_core *core, enum sp_udp_event event,
uint64_t now, uint8_t role) {
switch (event) {
case SP_UDP_BEGIN:
memset(core, 0, sizeof(*core));
core->role = role;
core->state = SP_UDP_MAPPING_ALLOCATED;
for (size_t i = 0; i < SP_UDP_PENDING_PER_FLOW; ++i)
core->pending_slots[i] = UINT16_MAX;
break;
case SP_UDP_NEGOTIATE:
core->state = SP_UDP_MAPPING_SOCKS_NEGOTIATING;
core->handshake_deadline_ns = sp_udp_deadline(now, SP_UDP_SOCKS_HANDSHAKE_SECONDS);
break;
case SP_UDP_READY:
core->state = SP_UDP_MAPPING_READY;
core->handshake_deadline_ns = 0;
core->idle_deadline_ns = sp_udp_deadline(now, SP_UDP_IDLE_SECONDS);
break;
case SP_UDP_PROGRESS:
core->idle_deadline_ns = sp_udp_deadline(now, SP_UDP_IDLE_SECONDS);
break;
case SP_UDP_EXPIRE:
if ((core->handshake_deadline_ns == 0 || now < core->handshake_deadline_ns) &&
(core->idle_deadline_ns == 0 || now < core->idle_deadline_ns))
return 0;
/* fall through */
case SP_UDP_CLOSE:
core->state = SP_UDP_MAPPING_DRAINING;
return 1;
}
return 0;
}
static int sp_udp_queue(struct sp_udp_core *core, uint16_t slot) {
if (core->pending_count == SP_UDP_PENDING_PER_FLOW)
return 0;
size_t tail = (core->pending_head + core->pending_count) % SP_UDP_PENDING_PER_FLOW;
core->pending_slots[tail] = slot;
core->pending_count++;
return 1;
}
/* Retire only accepted datagrams; zero-length payloads are still one handle. */
static void sp_udp_pop(struct sp_udp_core *core) {
core->pending_slots[core->pending_head] = UINT16_MAX;
core->pending_head = (uint8_t)((core->pending_head + 1U) % SP_UDP_PENDING_PER_FLOW);
core->pending_count--;
}
/* End Sans-I/O UDP. */
struct publish_udp_flow {
struct sp_udp_core core;
uint32_t generation;
uint8_t used;
uint8_t admission_held;
uint8_t global_admission_held;
uint8_t rule_admission_held;
int rule_index;
struct sockaddr_in host_peer;
uint32_t published_local_ip;
uint16_t synthetic_port;
uint16_t child_port;
};
static struct publish_udp_flow publish_udp_flows[MAX_PUBLISH_UDP];
static uint16_t next_publish_synth_port = PUBLISH_SYNTH_PORT_MIN;
struct dns_mapping {
uint8_t used;
uint8_t reserved[3];
uint32_t generation;
uint32_t synthetic_ip; /* network byte order */
uint64_t last_used_ns;
uint64_t expires_ns;
uint32_t active_refs;
uint16_t name_len;
char name[SP_DNS_NAME_MAX + 1];
};
struct sp_dns_core {
struct dns_mapping mappings[MAX_DNS_MAPPINGS];
uint32_t next_host;
uint32_t next_generation;
uint64_t ttl_ns;
};
enum sp_dns_output_credit {
SP_DNS_CREDIT_AVAILABLE = 0,
SP_DNS_CREDIT_NO_OUTPUT,
};
enum sp_dns_effect_kind {
SP_DNS_EFFECT_NONE = 0,
SP_DNS_EFFECT_RESPONSE,
SP_DNS_EFFECT_DROP,
SP_DNS_EFFECT_RETRY,
};
/* The query owns only the bounded question needed to reproduce the response.
* Additional bytes (for example an EDNS section) are deliberately not carried
* into the transition because this proxy has always answered one question. */
struct sp_dns_query {
uint16_t identifier;
uint16_t request_flags;
uint16_t qtype;
uint16_t qclass;
uint16_t question_len;
uint16_t name_len;
char name[SP_DNS_NAME_MAX + 1];
uint8_t question[SP_DNS_QUERY_MAX - 12];
};
struct sp_dns_response {
uint16_t len;
uint8_t bytes[SP_DNS_RESPONSE_MAX];
};
/* A mapping identity is deliberately smaller than its value. The slot is
* stable while the mapping is live; generation prevents a released handle
* from addressing a later occupant of the same slot. */
struct sp_dns_ref {
uint16_t slot;
uint32_t generation;
};
/* Borrowed while ref remains held. No copy is made for synchronous SOCKS
* encoding; asynchronous handshakes copy the bounded name into their owner.
* sp_dns_core_mapping_name rejects an otherwise valid but unpinned ref. */
struct sp_dns_name_view {
uint32_t synthetic_ip;
uint16_t name_len;
const char *name;
};
enum sp_dns_lookup_kind {
SP_DNS_LOOKUP_EXPIRE = 0,
SP_DNS_LOOKUP_NAME,
SP_DNS_LOOKUP_IP,
};
struct sp_dns_lookup_key {
enum sp_dns_lookup_kind kind;
const char *name;
uint16_t name_len;
uint32_t synthetic_ip;
};
struct sp_dns_lookup_result {
uint16_t match_slot;
uint16_t free_slot;
uint16_t oldest_unpinned_slot;
};
struct sp_dns_core_input {
struct sp_dns_query query;
uint64_t now_ns;
enum sp_dns_output_credit credit;
};
struct sp_dns_core_output {
uint8_t mapping_exhausted;
struct sp_dns_response response;
};
SP_TEST_API static int sp_dns_decode_query(const uint8_t *query, size_t len,
struct sp_dns_query *out);
SP_TEST_API static enum sp_dns_effect_kind
sp_dns_core_step(struct sp_dns_core *core,
const struct sp_dns_core_input *input,
struct sp_dns_core_output *out);
SP_TEST_API static int sp_dns_core_acquire_ip(
struct sp_dns_core *core, uint32_t synthetic_ip, uint64_t now_ns,
struct sp_dns_ref *out);
SP_TEST_API static int sp_dns_core_release(
struct sp_dns_core *core, struct sp_dns_ref ref, uint64_t now_ns);
SP_TEST_API static int sp_dns_core_mapping_name(
const struct sp_dns_core *core, struct sp_dns_ref ref,
struct sp_dns_name_view *out);
static struct sp_dns_core g_dns_core;
/* Check if IP is in gateway range (10.0.1.0/24) */
static int is_gateway_ip(uint32_t ip) {
return (ip & HOST_GATEWAY_MASK) == HOST_GATEWAY_BASE;
}
/* Extract last octet from gateway IP (10.0.1.x -> x) */
static uint8_t gateway_last_octet(uint32_t ip) {
return (uint8_t)((ip >> 24) & 0xff);
}
/* Convert gateway IP to localhost (10.0.1.x -> 127.0.0.x) */
static uint32_t gateway_to_localhost(uint32_t gw_ip) {
uint8_t last = gateway_last_octet(gw_ip);
return LOCALHOST_BASE | ((uint32_t)last << 24);
}
/* Check if gateway access is allowed for given IP, port, and protocol */
static int sp_policy_gateway_allowed(const struct sp_net_policy_snapshot *policy,
uint32_t gw_ip, uint16_t port,
int proto) {
if (policy == NULL || !policy->initialized)
return 0;
if (!is_gateway_ip(gw_ip))
return 0;
uint8_t last = gateway_last_octet(gw_ip);
for (uint16_t i = 0; i < policy->host_rule_count; i++) {
const struct host_rule *r = &policy->host_rules[i];
if (r->last_octet == last && r->port == port && (r->proto == 0 || r->proto == proto))
return 1;
}
return 0;
}
static uint32_t cidr_mask_from_prefix(unsigned int prefix) {
if (prefix == 0)
return 0;
return 0xffffffffU << (32U - prefix);
}
static int host_ip_in_cidr(uint32_t ip, uint32_t network, uint32_t mask) {
return (ip & mask) == network;
}
static int direct_ip_blocked_by_default(uint32_t ip_be) {
static const struct {
uint32_t network;
uint32_t mask;
} blocked[] = {
{0x00000000U, 0xff000000U}, /* 0.0.0.0/8 */
{0x0a000000U, 0xff000000U}, /* 10.0.0.0/8 */
{0x64400000U, 0xffc00000U}, /* 100.64.0.0/10 */
{0x7f000000U, 0xff000000U}, /* 127.0.0.0/8 */
{0xa9fe0000U, 0xffff0000U}, /* 169.254.0.0/16 */
{0xac100000U, 0xfff00000U}, /* 172.16.0.0/12 */
{0xc0000000U, 0xffffff00U}, /* 192.0.0.0/24 */
{0xc0000200U, 0xffffff00U}, /* 192.0.2.0/24 */
{0xc0a80000U, 0xffff0000U}, /* 192.168.0.0/16 */
{0xc6120000U, 0xfffe0000U}, /* 198.18.0.0/15 */
{0xc6336400U, 0xffffff00U}, /* 198.51.100.0/24 */
{0xcb007100U, 0xffffff00U}, /* 203.0.113.0/24 */
{0xe0000000U, 0xf0000000U}, /* 224.0.0.0/4 */
{0xf0000000U, 0xf0000000U}, /* 240.0.0.0/4 */
};
uint32_t ip = ntohl(ip_be);
for (size_t i = 0; i < sizeof(blocked) / sizeof(blocked[0]); ++i) {
if (host_ip_in_cidr(ip, blocked[i].network, blocked[i].mask))
return 1;
}
return 0;
}
static int direct_rule_matches(const struct direct_rule *rule, uint32_t ip_be,
uint16_t port, int proto) {
uint32_t ip = ntohl(ip_be);
return host_ip_in_cidr(ip, rule->network, rule->mask) &&
rule->port == port && (rule->proto == 0 || rule->proto == proto);
}
static int
sp_policy_direct_egress_allowed(const struct sp_net_policy_snapshot *policy,
uint32_t ip_be, uint16_t port, int proto) {
if (policy == NULL || !policy->initialized)
return 0;
if (!direct_ip_blocked_by_default(ip_be))
return 1;
for (uint16_t i = 0; i < policy->direct_rule_count; ++i) {
if (direct_rule_matches(&policy->direct_rules[i], ip_be, port, proto))
return 1;
}
return 0;
}
static void log_direct_egress_block(uint32_t ip_be, uint16_t port, int proto) {
char addr[INET_ADDRSTRLEN];
struct in_addr in = {.s_addr = ip_be};
const char *proto_name = proto == IPPROTO_TCP ? "tcp"
: proto == IPPROTO_UDP ? "udp"
: "ip";
if (inet_ntop(AF_INET, &in, addr, sizeof(addr)) == NULL)
snprintf(addr, sizeof(addr), "unknown");
DBG("[parent] direct %s egress blocked: %s:%u", proto_name, addr, port);
}
static enum parent_harden_profile
parent_harden_profile_for_egress(enum egress_mode mode) {
if (mode == EGRESS_SOCKS)
return PARENT_HARDEN_SOCKS;
if (mode == EGRESS_NONE)
return PARENT_HARDEN_EGRESS_NONE;
return PARENT_HARDEN_DIRECT;
}
static const char *
parent_harden_profile_name(enum parent_harden_profile profile) {
switch (profile) {
case PARENT_HARDEN_SOCKS:
return "socks";
case PARENT_HARDEN_EGRESS_NONE:
return "egress-none";
case PARENT_HARDEN_DIRECT:
default:
return "direct";
}
}
static struct parent_harden_config parent_harden_config_from_runtime(void) {
struct parent_harden_config cfg = {
.profile = parent_harden_profile_for_egress(egress_mode),
.outbound_sockets_allowed = egress_mode != EGRESS_NONE,
.interactive = interactive_stdio,
.child_pid = 0,
.child_pidfd = -1,
};
return cfg;
}
static void
parent_harden_log_prepared(const struct parent_harden_config *cfg) {
DBG("parent hardening prepared: profile=%s outbound_sockets=%s "
"interactive=%s setup_fds_closed=yes scratch_cleaned=yes "
"landlock=pending seccomp=pending",
parent_harden_profile_name(cfg->profile),
cfg->outbound_sockets_allowed ? "yes" : "no",
cfg->interactive ? "yes" : "no");
}
static void parent_harden_log_active(const struct parent_harden_config *cfg) {
DBG("parent hardening active: profile=%s outbound_sockets=%s "
"interactive=%s",
parent_harden_profile_name(cfg->profile),
cfg->outbound_sockets_allowed ? "yes" : "no",
cfg->interactive ? "yes" : "no");
}
#define MAX_CONNECTS_PER_SEC 50
#define TCP_HALF_OPEN_TIMEOUT_SEC 10
#define TCP_IDLE_TIMEOUT_SEC 120
struct udp_flow {
struct sp_udp_core core;
uint32_t generation;
uint8_t used;
uint8_t epoll_failed; /* an interest update failed; closed by the loop */
uint8_t admission_held;
uint8_t global_admission_held;
uint32_t cli_ip;
uint16_t cli_port;
uint32_t srv_ip;
uint16_t srv_port;
struct sockaddr_in connected_remote;
int tcp_ctrl; /* SOCKS5 TCP control connection (must stay open) */
int udp_relay; /* UDP socket to SOCKS relay */
int udp_staging; /* UDP socket bound before SOCKS UDP ASSOCIATE completes */
struct epoll_event relay_interest, ctrl_interest;
int effective_sndbuf;
int effective_rcvbuf;
struct sockaddr_in relay_addr; /* Expected relay source for validation */
struct socks_io socks;
uint8_t has_last_quote;
uint8_t last_quote[SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES];
struct sp_dns_ref dns_mapping_ref;
unsigned long dropped_backpressure;
};
static struct udp_flow udp_flows[MAX_UDP];
static int sp_udp_runtime_step(struct sp_udp_core *core, enum sp_udp_event event,
uint64_t now, uint8_t role) {
int result = sp_udp_step(core, event, now, role);
uintptr_t address = (uintptr_t)core;
unsigned owner;
uint64_t deadline = core->handshake_deadline_ns;
if (address >= (uintptr_t)udp_flows && address < (uintptr_t)(udp_flows + MAX_UDP))
owner = SP_DEADLINE_UDP + (unsigned)((address - (uintptr_t)udp_flows) / sizeof(*udp_flows));
else if (address >= (uintptr_t)publish_udp_flows &&
address < (uintptr_t)(publish_udp_flows + MAX_PUBLISH_UDP))
owner = SP_DEADLINE_PUBLISH + (unsigned)((address - (uintptr_t)publish_udp_flows) / sizeof(*publish_udp_flows));
else
return result;
if (deadline == 0 || (core->idle_deadline_ns && core->idle_deadline_ns < deadline))
deadline = core->idle_deadline_ns;
sp_deadline_set(&g_net_deadlines, owner, result ? 0 : deadline);
return result;
}
struct sp_udp_pending_slot {
uint32_t generation;
uint16_t length;
uint16_t owner_index;
uint8_t used;
uint8_t owner_role;
uint8_t owner_publish;
uint8_t has_quote;
uint8_t quote[SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES];
};
static struct sp_udp_pending_slot
g_udp_pending_slots[SP_UDP_PENDING_ALL];
enum {
SP_UDP_MMSG_RX_BYTES = 2048,
SP_UDP_MMSG_CONTROL_BYTES = 2 * CMSG_SPACE(sizeof(struct in_pktinfo)),
};
/* One broker thread owns these reusable parallel vectors. mmsghdr must be a
* genuinely contiguous array for the syscall ABI. Every submitted index is
* fully reset immediately before recvmmsg(), so a short vector cannot expose
* name, control, flag, length, or payload metadata from an earlier turn. */
static struct mmsghdr g_udp_mmsg_messages[SP_UDP_MMSG_MAX];
static struct iovec g_udp_mmsg_iov[SP_UDP_MMSG_MAX];
static struct sockaddr_in g_udp_mmsg_sources[SP_UDP_MMSG_MAX];
static unsigned char
g_udp_mmsg_control[SP_UDP_MMSG_MAX][SP_UDP_MMSG_CONTROL_BYTES];
static uint8_t g_udp_mmsg_payload[SP_UDP_MMSG_MAX][SP_UDP_MMSG_RX_BYTES];
enum {
SP_UDP_MMSG_TX_BYTES =
SP_UDP_PENDING_PER_FLOW * SP_UDP_PAYLOAD_MAX + 256,
};
static struct mmsghdr g_udp_mmsg_tx_messages[SP_UDP_MMSG_MAX];
static struct iovec g_udp_mmsg_tx_iov[SP_UDP_MMSG_MAX];
static struct sockaddr_in g_udp_mmsg_tx_names[SP_UDP_MMSG_MAX];
static unsigned char
g_udp_mmsg_tx_control[SP_UDP_MMSG_MAX][SP_UDP_MMSG_CONTROL_BYTES];
static uint8_t g_udp_mmsg_tx_payload[SP_UDP_MMSG_MAX][SP_UDP_MMSG_TX_BYTES];
static size_t g_udp_pending_live_bytes;
static uint64_t g_ns_aux_hash_key;
static uint64_t sp_ns_now_ns(void);
/* I/O buffer for event loop reads - safe as static since writes are via
opaque syscalls (read/recv) that act as compiler barriers. */
static uint8_t g_io_buf[65536];
struct sp_resource_geometry {
size_t current_static_arena_bytes;
size_t declared_table_bytes;
size_t declared_userspace_pool_bytes;
size_t requested_kernel_socket_bytes;
size_t declared_total_envelope_bytes;
size_t selected_fd_ceiling;
size_t k_rx;
};
struct sp_tcp_key {
uint32_t cli_ip, srv_ip;
uint16_t cli_port, srv_port;
};
_Static_assert(sizeof(struct sp_tcp_key) == 12, "TCP tuple has no padding");
struct sp_timewait_key {
uint32_t cli_ip, srv_ip;
uint16_t cli_port, srv_port;
};
_Static_assert(sizeof(struct sp_timewait_key) == 12, "TIME-WAIT tuple has no padding");
struct sp_udp_key { uint8_t bytes[24]; };
struct sp_udp_table_value { uint16_t index; uint8_t role; };
struct sp_udp_entry {
struct sp_udp_key key;
uint32_t generation;
struct sp_udp_table_value value;
};
struct sp_resource_admission_value { uint64_t allowance, last_ns; };
struct sp_admission_entry {
struct sp_admission_key key;
bool live;
uint64_t refs;
struct sp_resource_admission_value value;
};
enum sp_ns_timewait_slot_state { SP_NS_TIMEWAIT_EMPTY, SP_NS_TIMEWAIT_LIVE };
#define SP_NS_TIMEWAIT_ADMISSION_REF UINT16_C(1)
struct sp_ns_timewait_entry {
uint64_t expiry_ns, snd_nxt_abs, rcv_acked_abs, rcv_adv_right_abs;
uint32_t cli_ip, srv_ip;
uint16_t cli_port, srv_port;
uint8_t state, role;
uint16_t reserved;
uint32_t admission_source_ip;
};
struct sp_tcp_map {
struct ht_index index;
struct ht_hash_key secret;
struct sp_ns_flow *flows;
};
struct sp_udp_map {
struct ht_index index;
struct ht_hash_key secret;
struct sp_udp_entry *entries;
uint32_t *free_slots, free_count;
};
struct sp_timewait_map {
struct ht_index index;
struct ht_hash_key secret;
struct sp_ns_timewait_entry *entries;
uint32_t *free_slots, free_count;
};
struct sp_admission_map {
struct ht_index index;
struct ht_hash_key secret;
struct sp_admission_entry *entries;
uint32_t *free_slots, free_count;
};
/* One compile-time arena layout: mmap supplies page alignment, C supplies typed
* offsets. No general allocator, resizing or runtime allocation interface. */
struct sp_map_storage {
_Alignas(64) struct ht_bucket tcp[SP_TCP_TABLE_SLOTS];
_Alignas(64) struct ht_bucket udp[SP_UDP_TABLE_SLOTS];
_Alignas(64) struct ht_bucket timewait[SP_TIMEWAIT_TABLE_SLOTS];
_Alignas(64) struct ht_bucket admission[SP_ADMISSION_TABLE_SLOTS];
struct sp_udp_entry udp_entries[SP_UDP_KEY_MAX];
struct sp_ns_timewait_entry timewait_entries[SP_TIMEWAIT_ALL_MAX];
struct sp_admission_entry admission_entries[SP_ADMISSION_KEY_MAX];
uint32_t udp_free[SP_UDP_KEY_MAX];
uint32_t timewait_free[SP_TIMEWAIT_ALL_MAX];
uint32_t admission_free[SP_ADMISSION_KEY_MAX];
};
static struct sp_map_storage *g_map_storage;
static struct sp_tcp_map g_tcp_map;
static struct sp_udp_map g_udp_map;
static struct sp_timewait_map g_timewait_map;
static struct sp_admission_map g_admission_map;
/* Fixed packet arenas stay in BSS and become resident only when their owning
* path writes them. Startup initializes queue metadata, not every payload
* page: a configured upper bound is not a reason to charge idle RSS. */
static uint8_t g_resource_udp_pending_arena[SP_UDP_PENDING_BYTES];
static uint8_t g_resource_tun_data_arena[SP_TUN_DATA_MAX_BYTES];
static uint8_t g_resource_tun_control_arena[SP_TUN_CONTROL_MAX_BYTES];
enum { SP_NS_QUEUE_MAX = 16, SP_NS_PUMP_ITERS = SP_EFFECTS_TURN };
static int g_ns_tunfd_for_emit = -1;
enum sp_pump_item_kind {
SP_PUMP_ITEM_ADMISSION = 1,
SP_PUMP_ITEM_SEGMENT,
SP_PUMP_ITEM_CONNECT_RESULT,
SP_PUMP_ITEM_STREAM_SEND_RESULT,
SP_PUMP_ITEM_STREAM_CONSUMED,
SP_PUMP_ITEM_STREAM_EOF,
SP_PUMP_ITEM_TUN_EMIT_RESULT,
SP_PUMP_ITEM_CREDIT,
SP_PUMP_ITEM_TIMER_FIRED,
SP_PUMP_ITEM_INGRESS_DISPOSED,
};
struct sp_pump_item {
enum sp_pump_item_kind kind;
uint64_t now; /* copied from the event's context when the item is built */
union {
struct sp_ev_flow_admitted admission;
struct sp_ev_tcp_segment segment;
struct sp_ev_connect_result connect_result;
struct sp_ev_stream_send_result stream_send_result;
struct sp_ev_stream_consumed stream_consumed;
struct sp_ev_stream_eof stream_eof;
struct sp_ev_tun_emit_result tun_emit_result;
struct sp_ev_credit credit;
struct sp_ev_timer_fired timer_fired;
struct sp_ev_ingress_disposed ingress_disposed;
} value;
};
static enum sp_tcp_core_result sp_pump_item_dispatch(
struct sp_tcp_core *core, const struct sp_pump_item *item,
struct sp_typed_effect_list *effects);
_Static_assert(sizeof(struct sp_pump_item) <= 104,
"private pump item exceeded its reviewed D3 ceiling");
/* One bounded workspace; no pump continuation survives a call. */
struct sp_ns_pump {
struct sp_pump_item queue[SP_NS_QUEUE_MAX];
size_t count;
struct sp_ns_flow *owner;
int tunfd;
};
static struct sp_ns_pump g_ns_pump;
static uint32_t g_ns_timewait_heap_count;
static int sp_checked_add_size(size_t left, size_t right, size_t *out) {
if (out == NULL || left > SIZE_MAX - right)
return -1;
*out = left + right;
return 0;
}
static int sp_checked_mul_size(size_t left, size_t right, size_t *out) {
if (out == NULL || (right != 0 && left > SIZE_MAX / right))
return -1;
*out = left * right;
return 0;
}
static int sp_checked_ceil_div_size(size_t numerator, size_t denominator,
size_t *out) {
size_t adjusted;
if (out == NULL || denominator == 0)
return -1;
if (numerator == 0) {
*out = 0;
return 0;
}
if (sp_checked_add_size(numerator, denominator - 1, &adjusted) < 0)
return -1;
*out = adjusted / denominator;
return 0;
}
static int sp_resource_compute_geometry(int selected_tcp, int selected_udp,
int selected_publish_rules,
struct sp_resource_geometry *geometry) {
size_t total = 0;
size_t value;
size_t stream_fds;
size_t datagram_fds;
size_t socks_control_fds = SP_UDP_SOCKS_MAX;
size_t publish_listener_fds;
size_t fixed_fds = SP_FIXED_AUTHORITY_FDS + SP_CONNECT_INFLIGHT_MAX;
extern char __executable_start;
extern char _end;
if (geometry == NULL || selected_tcp < 1 || selected_tcp > MAX_TCP ||
selected_udp < 1 || selected_udp > MAX_UDP ||
selected_publish_rules < 0 ||
selected_publish_rules > MAX_PUBLISH_RULES)
return -1;
memset(geometry, 0, sizeof(*geometry));
if ((uintptr_t)&_end < (uintptr_t)&__executable_start)
return -1;
/* Conservative ELF image span: includes code/rodata and alignment as well
* as writable pools. __data_start is a glibc CRT symbol absent on musl. */
geometry->current_static_arena_bytes =
(size_t)((uintptr_t)&_end - (uintptr_t)&__executable_start);
geometry->declared_table_bytes =
sizeof(struct sp_map_storage);
geometry->declared_userspace_pool_bytes =
geometry->current_static_arena_bytes + geometry->declared_table_bytes;
if (sp_checked_ceil_div_size(SP_TCP_PENDING_PER_FLOW,
SP_TCP_GSO_POLICY_SEGS *
SP_TCP_MTU1500_MAX_MSS,
&geometry->k_rx) < 0 ||
geometry->k_rx != 6 || SP_INGRESS_STAGE_SLOTS < 2 * geometry->k_rx)
return -1;
total = 0;
stream_fds = (size_t)selected_tcp;
if (sp_checked_mul_size((size_t)selected_publish_rules, 2, &value) < 0 ||
sp_checked_add_size((size_t)selected_udp, socks_control_fds,
&datagram_fds) < 0 ||
sp_checked_add_size(stream_fds, datagram_fds, &total) < 0 ||
sp_checked_add_size(total, value, &total) < 0 ||
sp_checked_add_size(total, fixed_fds, &total) < 0)
return -1;
geometry->selected_fd_ceiling = total;
publish_listener_fds = (size_t)selected_publish_rules;
if (sp_checked_add_size(stream_fds, socks_control_fds, &stream_fds) < 0 ||
sp_checked_add_size(stream_fds, publish_listener_fds, &stream_fds) < 0 ||
sp_checked_add_size((size_t)selected_udp, publish_listener_fds,
&datagram_fds) < 0)
return -1;
total = 0;
if (sp_checked_mul_size(stream_fds,
2U * (SP_HOST_TCP_SNDBUF + SP_HOST_TCP_RCVBUF),
&total) < 0 ||
sp_checked_mul_size(datagram_fds,
2U * (SP_HOST_UDP_SNDBUF + SP_HOST_UDP_RCVBUF),
&value) < 0 ||
sp_checked_add_size(total, value, &total) < 0)
return -1;
geometry->requested_kernel_socket_bytes = total;
value = geometry->declared_userspace_pool_bytes;
if (sp_checked_add_size(value, geometry->requested_kernel_socket_bytes,
&value) < 0 ||
sp_checked_add_size(value, SP_RESOURCE_MEMORY_MARGIN_BYTES, &value) < 0)
return -1;
geometry->declared_total_envelope_bytes = value;
return 0;
}
static int sp_resource_validate_startup(void) {
struct sp_resource_geometry geometry;
struct rlimit nofile;
if (sp_resource_compute_geometry(tcp_flow_limit, udp_flow_limit,
publish_rule_count, &geometry) < 0) {
fprintf(stderr, "[sockpuppet] invalid resource geometry\n");
return -1;
}
if (getrlimit(RLIMIT_NOFILE, &nofile) < 0)
return -1;
if (geometry.declared_total_envelope_bytes > SP_BROKER_MEMORY_MAX_BYTES) {
fprintf(stderr,
"[sockpuppet] declared memory envelope=%zu exceeds broker "
"MemoryMax=%zu\n",
geometry.declared_total_envelope_bytes,
SP_BROKER_MEMORY_MAX_BYTES);
errno = ENOMEM;
return -1;
}
if (nofile.rlim_cur != RLIM_INFINITY &&
geometry.selected_fd_ceiling > (size_t)nofile.rlim_cur) {
fprintf(stderr,
"[sockpuppet] RLIMIT_NOFILE=%llu cannot cover fd ceiling=%zu\n",
(unsigned long long)nofile.rlim_cur,
geometry.selected_fd_ceiling);
errno = EMFILE;
return -1;
}
DBG("resource geometry: static=%zu pools=%zu kernel_socket_request=%zu fds=%zu "
"k_rx=%zu",
geometry.current_static_arena_bytes,
geometry.declared_userspace_pool_bytes,
geometry.requested_kernel_socket_bytes, geometry.selected_fd_ceiling,
geometry.k_rx);
return 0;
}
static void sp_resource_print_counters(void) {
(void)sp_stderr_printf("sp_runtime_schema\tcounters-v2\n");
(void)sp_stderr_printf(
"sp_runtime_config\tvnet_profile_validated\t%d\t"
"header_bytes\t%d\trequested_offloads\tCSUM|TSO4\t"
"gso_max_size\t%d\tgso_max_segs\t%d\n",
g_vnet_profile_validated,
SP_VNET_HDR_BYTES, SP_TUN_BODY_MAX_BYTES, SP_TCP_GSO_POLICY_SEGS);
#define SP_RESOURCE_COUNTER_PRINT(name) \
(void)sp_stderr_printf("sp_runtime_counter\t" #name "\t%llu\n", \
(unsigned long long)g_resource_counters.name);
SP_RESOURCE_COUNTER_SCHEMA(SP_RESOURCE_COUNTER_PRINT)
#undef SP_RESOURCE_COUNTER_PRINT
}
/* ---------- utilities ---------- */
static int sp_is_namespace_child;
static void die(const char *msg) {
perror(msg);
if (sp_is_namespace_child)
_exit(1);
exit(1);
}
struct sp_child_owner {
pid_t pid;
int pidfd;
};
static int sp_take_fd(int *fd) {
int taken;
if (fd == NULL)
return -1;
taken = *fd;
*fd = -1;
return taken;
}
static int sp_close_owned_fd(int *fd) {
int owned;
if (fd == NULL || *fd < 0)
return 0;
owned = sp_take_fd(fd);
return close(owned);
}
static void sp_close_owned_fd_nointr(int *fd) {
(void)sp_close_owned_fd(fd);
}
static void sp_child_owner_init(struct sp_child_owner *child) {
child->pid = -1;
child->pidfd = -1;
}
static int sp_open_directory_nosymlinks(const char *path) {
struct open_how how = {
.flags = O_PATH | O_DIRECTORY | O_CLOEXEC,
.resolve = RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS,
};
return (int)syscall(SYS_openat2, AT_FDCWD, path, &how, sizeof(how));
}
static int sp_open_directory_for_lock(const char *path) {
struct open_how how = {
.flags = O_RDONLY | O_DIRECTORY | O_CLOEXEC,
.resolve = RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS,
};
return (int)syscall(SYS_openat2, AT_FDCWD, path, &how, sizeof(how));
}
static int sp_open_directory_beneath(int root_fd,
const char *absolute_path) {
struct open_how how = {
.flags = O_PATH | O_DIRECTORY | O_CLOEXEC,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS |
RESOLVE_NO_SYMLINKS,
};
return (int)syscall(SYS_openat2, root_fd, absolute_path + 1, &how,
sizeof(how));
}
static void sp_prepared_fs_init(struct sp_prepared_fs *prepared) {
memset(prepared, 0, sizeof(*prepared));
prepared->retained_lock_fd = -1;
prepared->scratch_parent_fd = -1;
}
static int sp_prepared_fs_cleanup_scratch(struct sp_prepared_fs *prepared) {
if (!prepared->scratch_owned)
return 0;
if (unlinkat(prepared->scratch_parent_fd, prepared->scratch_name,
AT_REMOVEDIR) < 0 && errno != ENOENT)
return -1;
prepared->scratch_owned = 0;
return 0;
}
static void sp_prepared_fs_close_all(struct sp_prepared_fs *prepared) {
(void)sp_prepared_fs_cleanup_scratch(prepared);
sp_close_owned_fd_nointr(&prepared->retained_lock_fd);
sp_close_owned_fd_nointr(&prepared->scratch_parent_fd);
}
static void
sp_prepared_fs_close_child_scratch(struct sp_prepared_fs *prepared) {
prepared->scratch_owned = 0;
sp_close_owned_fd_nointr(&prepared->scratch_parent_fd);
}
static int sp_prepare_fs_storage(struct sp_prepared_fs *prepared,
const struct sp_fs_config *config,
char *overlay_base, size_t overlay_base_size,
char *error, size_t error_size) {
char scratch[] = "/tmp/.sockpuppet-overlay-XXXXXX";
const char *name;
if (config->retained_directory != NULL) {
prepared->retained_lock_fd =
sp_open_directory_for_lock(config->retained_directory);
if (prepared->retained_lock_fd < 0) {
snprintf(error, error_size, "open retained directory");
return -1;
}
if (flock(prepared->retained_lock_fd, LOCK_EX | LOCK_NB) < 0) {
snprintf(error, error_size, "lock retained directory");
sp_prepared_fs_close_all(prepared);
return -1;
}
}
if (mkdtemp(scratch) == NULL) {
snprintf(error, error_size, "create overlay scratch directory");
sp_prepared_fs_close_all(prepared);
return -1;
}
name = strrchr(scratch, '/');
name = name == NULL ? scratch : name + 1;
if (snprintf(prepared->scratch_name, sizeof(prepared->scratch_name), "%s",
name) >= (int)sizeof(prepared->scratch_name) ||
snprintf(overlay_base, overlay_base_size, "%s", scratch) >=
(int)overlay_base_size) {
(void)rmdir(scratch);
errno = ENAMETOOLONG;
snprintf(error, error_size, "overlay scratch path too long");
sp_prepared_fs_close_all(prepared);
return -1;
}
prepared->scratch_parent_fd =
open("/tmp", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (prepared->scratch_parent_fd < 0) {
int saved = errno;
(void)rmdir(scratch);
errno = saved;
snprintf(error, error_size, "open overlay scratch parent");
sp_prepared_fs_close_all(prepared);
return -1;
}
prepared->scratch_owned = 1;
return 0;
}
static void sp_kill_and_reap_child(struct sp_child_owner *child,
int report_fatal) {
int saved = errno;
int killed = 0;
if (child->pid <= 0) {
sp_close_owned_fd_nointr(&child->pidfd);
errno = saved;
return;
}
if (report_fatal)
fprintf(stderr,
"[sockpuppet] Error: fatal broker error; killing sandbox child\n");
#ifdef __NR_pidfd_send_signal
if (child->pidfd >= 0 &&
syscall(__NR_pidfd_send_signal, child->pidfd, SIGKILL, NULL, 0) == 0)
killed = 1;
#endif
if (!killed && (kill(child->pid, SIGKILL) == 0 || errno == ESRCH))
killed = 1;
if (!killed) {
if (report_fatal)
fprintf(stderr,
"[sockpuppet] Warning: could not kill sandbox child: %s; "
"leaving it unreaped\n",
strerror(errno));
errno = saved;
return;
}
for (;;) {
pid_t reaped = waitpid(child->pid, NULL, 0);
if (reaped == child->pid || (reaped < 0 && errno == ECHILD)) {
child->pid = -1;
break;
}
if (reaped < 0 && errno == EINTR)
continue;
break;
}
(void)sp_close_owned_fd(&child->pidfd);
errno = saved;
}
static ssize_t write_all(int fd, const void *buf, size_t len) {
const uint8_t *p = buf;
size_t off = 0;
while (off < len) {
ssize_t w = write(fd, p + off, len - off);
if (w > 0) {
off += (size_t)w;
continue;
}
if (w < 0 && errno == EINTR)
continue;
if (w == 0)
errno = EIO;
return -1;
}
return (ssize_t)off;
}
static ssize_t read_all(int fd, void *buf, size_t len) {
uint8_t *p = buf;
size_t off = 0;
while (off < len) {
ssize_t r = read(fd, p + off, len - off);
if (r > 0) {
off += (size_t)r;
continue;
}
if (r < 0 && errno == EINTR)
continue;
if (r == 0)
errno = EPIPE;
return -1;
}
return (ssize_t)off;
}
/* Failed epoll interest updates (F2). An update that fails leaves the kernel
* watching a different set of events than the broker intends, which can
* stall a flow or spin on stale level-triggered readiness. The owner of the
* registration decides: a TCP flow is reset, a UDP flow is marked and closed
* by the broker loop before its next wait (its update helpers have callers
* that keep using the flow), and a broker-owned registration (TUN, publish
* listener) makes the broker loop fail. Fatal scheduler TUN writes use the
* same broker failure latch; they never become retryable backpressure. */
static int g_broker_io_failed;
static int g_udp_epoll_failed;
static void sp_epoll_update_failed(const char *what) {
sp_counter_inc(&g_resource_counters.epoll_update_failures);
DBG("epoll_ctl mod %s: %s", what, strerror(errno));
}
static void tun_update_events(int tunfd) {
struct epoll_event ev = {
.events = EPOLLIN |
((g_tun_queue.count > 0 || g_ns_sched_tun_pending) ? EPOLLOUT
: 0),
.data.u64 = epoll_token_encode(FD_TUN, 0, (uint16_t)tunfd, 1),
};
if (g_epfd >= 0 && g_tun_registered_fd == tunfd &&
sp_epoll_update(EPOLL_CTL_MOD, tunfd, &ev, &g_tun_interest) < 0) {
sp_epoll_update_failed("TUN");
g_broker_io_failed = 1;
}
}
static int tun_queue_packet(int tunfd, const uint8_t *buf, size_t len,
const char *what) {
size_t tail;
if (len > TUN_QUEUE_PACKET_MAX || g_tun_queue.count == TUN_QUEUE_PACKETS) {
sp_counter_inc(&g_resource_counters.tun_queue_drops);
DBG("TUN output queue full (%s), dropping packet", what);
errno = len > TUN_QUEUE_PACKET_MAX ? EMSGSIZE : EAGAIN;
return -1;
}
tail = (g_tun_queue.head + g_tun_queue.count) % TUN_QUEUE_PACKETS;
g_tun_queue.packets[tail].len = len;
memcpy(g_tun_queue.packets[tail].data, buf, len);
g_tun_queue.count++;
if (g_resource_metrics_enabled &&
g_tun_queue.count > g_resource_counters.tun_queue_high_water)
g_resource_counters.tun_queue_high_water = g_tun_queue.count;
tun_update_events(tunfd);
return 0;
}
static int sp_vnet_output_envelope_valid(const uint8_t *record, size_t len);
static ssize_t tun_kernel_write(int tunfd, const uint8_t *record, size_t len) {
return write(tunfd, record, len);
}
static int tun_write_frame(int tunfd, const uint8_t *frame, size_t len,
const char *what) {
if (!sp_vnet_output_envelope_valid(frame, len)) {
DBG("invalid VNET output envelope (%s): %zu", what, len);
errno = EPROTO;
return -1;
}
if (g_tun_queue.count > 0)
return tun_queue_packet(tunfd, frame, len, what);
for (;;) {
ssize_t n = tun_kernel_write(tunfd, frame, len);
if (n == (ssize_t)len)
return 0;
if (n >= 0) {
DBG("TUN write short (%s): %zd/%zu, dropping frame", what, n, len);
errno = EIO;
return -1;
}
if (errno == EINTR)
continue;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return tun_queue_packet(tunfd, frame, len, what);
int saved = errno;
DBG("TUN write failed (%s): %s", what, strerror(saved));
errno = saved;
return -1;
}
}
/* Returns zero only when the packet is accepted by the kernel or the bounded
* queue. Callers that advance protocol state must use this as their commit
* point: a packet rejected because the queue is full was never emitted. */
static int tun_write_packet(int tunfd, const uint8_t *buf, size_t len,
const char *what) {
uint8_t frame[SP_VNET_HDR_BYTES + 1500];
if (len == 0 || len > 1500) {
DBG("invalid TUN packet length (%s): %zu", what, len);
return -1;
}
memset(frame, 0, SP_VNET_HDR_BYTES);
memcpy(frame + SP_VNET_HDR_BYTES, buf, len);
int result =
tun_write_frame(tunfd, frame, SP_VNET_HDR_BYTES + len, what);
if (result == 0)
sp_counter_inc(&g_resource_counters.tun_vnet_scalar_egress);
return result;
}
/* Drain queued packets on EPOLLOUT. TUN packet writes must be atomic; a
* short write cannot be resumed without corrupting the packet boundary. */
static int tun_flush_packets(int tunfd) {
while (g_tun_queue.count > 0) {
struct tun_packet *packet = &g_tun_queue.packets[g_tun_queue.head];
ssize_t n = tun_kernel_write(tunfd, packet->data, packet->len);
if (n == (ssize_t)packet->len) {
g_tun_queue.head = (g_tun_queue.head + 1) % TUN_QUEUE_PACKETS;
g_tun_queue.count--;
continue;
}
if (n < 0 && errno == EINTR)
continue;
if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
if (n > 0)
DBG("TUN queued write short: %zd/%zu, dropping packet", n,
packet->len);
else
DBG("TUN queued write failed: %s", strerror(errno));
return -1;
}
tun_update_events(tunfd);
return 0;
}
static void interactive_handle_sigwinch(int signo) {
(void)signo;
interactive_resize_pending = 1;
}
static int parse_long_strict(const char *s, long min, long max, long *out) {
char *end = NULL;
long value;
if (s == NULL || *s == '\0')
return -1;
errno = 0;
value = strtol(s, &end, 10);
if (errno != 0 || end == NULL || *end != '\0' || value < min || value > max)
return -1;
*out = value;
return 0;
}
static int sp_validate_absolute_path(const char *path, const char *what,
char *error, size_t error_size) {
const char *component;
if (path == NULL || path[0] != '/') {
snprintf(error, error_size, "%s must be an absolute path", what);
return -1;
}
if (path[1] == '\0')
return 0;
if (path[strlen(path) - 1] == '/') {
snprintf(error, error_size, "%s must not end with '/'", what);
return -1;
}
component = path + 1;
while (*component != '\0') {
const char *end = strchr(component, '/');
size_t length = end == NULL ? strlen(component) : (size_t)(end - component);
if (length == 0 || (length == 1 && component[0] == '.') ||
(length == 2 && component[0] == '.' && component[1] == '.')) {
snprintf(error, error_size,
"%s must not contain empty, '.' or '..' components", what);
return -1;
}
if (end == NULL)
break;
component = end + 1;
}
return 0;
}
static int sp_path_contains(const char *parent, const char *child) {
size_t length = strlen(parent);
if (strcmp(parent, child) == 0)
return 1;
if (strcmp(parent, "/") == 0)
return 1;
return strncmp(parent, child, length) == 0 && child[length] == '/';
}
static int sp_paths_overlap(const char *left, const char *right) {
return sp_path_contains(left, right) || sp_path_contains(right, left);
}
static int sp_bind_target_reserved(const char *target) {
static const char *const reserved[] = {
"/usr", "/lib", "/proc", "/dev", "/sys",
"/run", "/tmp", "/workspace", "/.oldroot",
};
if (strcmp(target, "/") == 0)
return 1;
for (size_t i = 0; i < sizeof(reserved) / sizeof(reserved[0]); ++i) {
if (sp_path_contains(reserved[i], target))
return 1;
}
return 0;
}
static int sp_parse_bind_spec(struct sp_bind_spec *out, const char *name,
const char *source, const char *target,
const char *mode, char *error,
size_t error_size) {
size_t name_length = strlen(name);
if (name_length == 0 || name_length > SP_BIND_NAME_MAX) {
snprintf(error, error_size, "bind name must contain 1-%d characters",
SP_BIND_NAME_MAX);
return -1;
}
for (size_t i = 0; i < name_length; ++i) {
unsigned char ch = (unsigned char)name[i];
if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') || ch == '_' || ch == '-')) {
snprintf(error, error_size,
"bind name must use ASCII letters, digits, '_' or '-'");
return -1;
}
}
if (sp_validate_absolute_path(source, "bind source", error, error_size) < 0 ||
sp_validate_absolute_path(target, "bind target", error, error_size) < 0)
return -1;
if (strcmp(mode, "ro") == 0)
out->mode = SP_BIND_RO;
else if (strcmp(mode, "rw") == 0)
out->mode = SP_BIND_RW;
else if (strcmp(mode, "cow") == 0)
out->mode = SP_BIND_COW;
else {
snprintf(error, error_size, "bind mode must be ro, rw or cow");
return -1;
}
memcpy(out->name, name, name_length + 1);
out->source = source;
out->target = target;
return 0;
}
static int sp_validate_bind_spec(const struct sp_bind_spec *bind,
const char *rootfs_path,
const char *retained_directory, char *error,
size_t error_size) {
if (sp_bind_target_reserved(bind->target)) {
snprintf(error, error_size, "bind target is in a reserved tree");
return -1;
}
if (rootfs_path != NULL && sp_paths_overlap(bind->target, rootfs_path)) {
snprintf(error, error_size, "bind target overlaps the rootfs path");
return -1;
}
if (retained_directory != NULL &&
sp_paths_overlap(bind->target, retained_directory)) {
snprintf(error, error_size, "bind target overlaps the retained directory");
return -1;
}
return 0;
}
static int sp_validate_launch_config(const struct sp_launch_config *config,
char *error, size_t error_size) {
const struct sp_fs_config *filesystem = &config->filesystem;
const struct sp_setup_config *setup = &config->setup;
int retained_view =
filesystem->rootfs_view == SP_VIEW_COW_RETAINED ||
filesystem->workspace_view == SP_VIEW_COW_RETAINED;
if (filesystem->rootfs_path != NULL &&
sp_validate_absolute_path(filesystem->rootfs_path, "rootfs", error,
error_size) < 0)
return -1;
if (filesystem->retained_directory != NULL &&
sp_validate_absolute_path(filesystem->retained_directory,
"overlay directory", error, error_size) < 0)
return -1;
if (retained_view != (filesystem->retained_directory != NULL)) {
snprintf(error, error_size,
"--overlay-dir is required exactly for retained views");
return -1;
}
if (filesystem->rootfs_path != NULL &&
filesystem->retained_directory != NULL &&
sp_paths_overlap(filesystem->rootfs_path,
filesystem->retained_directory)) {
snprintf(error, error_size, "rootfs overlaps the retained directory");
return -1;
}
if (filesystem->bind_enabled &&
sp_validate_bind_spec(&filesystem->bind, filesystem->rootfs_path,
filesystem->retained_directory, error,
error_size) < 0)
return -1;
if (setup->host_script != NULL && setup->guest_script != NULL) {
snprintf(error, error_size, "--setup and --setup-guest are mutually exclusive");
return -1;
}
if (setup->host_script != NULL &&
sp_validate_absolute_path(setup->host_script, "setup script", error,
error_size) < 0)
return -1;
if (setup->guest_script != NULL &&
sp_validate_absolute_path(setup->guest_script, "guest setup script", error,
error_size) < 0)
return -1;
if (setup->guest_script != NULL && filesystem->rootfs_path == NULL) {
snprintf(error, error_size, "--setup-guest requires --rootfs");
return -1;
}
if (setup->timeout_ms == 0 ||
setup->timeout_ms > SP_SETUP_TIMEOUT_MAX_MS) {
snprintf(error, error_size, "setup timeout must be between 1 and 300 seconds");
return -1;
}
if (setup->payload_uid == 0 || setup->payload_uid == (uid_t)-1 ||
setup->payload_gid == 0 || setup->payload_gid == (gid_t)-1) {
snprintf(error, error_size, "payload UID and GID must be nonzero");
return -1;
}
return 0;
}
/* Helper to suppress unused result warnings from FORTIFY_SOURCE */
#define IGNORE_RESULT(x) \
do { \
if (x) { \
} \
} while (0)
static int write_file_checked(const char *path, const char *data) {
int fd = open(path, O_WRONLY);
size_t len = strlen(data);
if (fd < 0)
return -1;
if (write_all(fd, data, len) != (ssize_t)len) {
int saved = errno;
close(fd);
errno = saved;
return -1;
}
if (close(fd) < 0)
return -1;
return 0;
}
static int set_rlimit_checked(int resource, rlim_t soft, rlim_t hard,
const char *name) {
struct rlimit lim = {.rlim_cur = soft, .rlim_max = hard};
if (setrlimit(resource, &lim) < 0)
return -1;
DBG("RLIMIT %s set to soft=%llu hard=%llu", name,
(unsigned long long)soft, (unsigned long long)hard);
return 0;
}
static int set_nofile_rlimit_checked(rlim_t requested) {
struct rlimit inherited;
rlim_t applied = requested;
if (getrlimit(RLIMIT_NOFILE, &inherited) < 0)
return -1;
if (inherited.rlim_max != RLIM_INFINITY && applied > inherited.rlim_max)
applied = inherited.rlim_max;
return set_rlimit_checked(RLIMIT_NOFILE, applied, applied, "NOFILE");
}
static int apply_parent_rlimits(void) {
return set_rlimit_checked(RLIMIT_CORE, 0, 0, "CORE") < 0 ||
set_rlimit_checked(RLIMIT_MEMLOCK, 0, 0, "MEMLOCK") < 0 ||
set_nofile_rlimit_checked(RLIMIT_PARENT_NOFILE) < 0
? -1
: 0;
}
__attribute__((format(printf, 1, 2))) static void
startup_warn(const char *fmt, ...) {
va_list ap;
fprintf(stderr, "[sockpuppet] Warning: ");
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
va_end(ap);
fputc('\n', stderr);
}
enum bench_trace_role {
BENCH_ROLE_UNKNOWN = 0,
BENCH_ROLE_PARENT,
BENCH_ROLE_PID1,
BENCH_ROLE_PAYLOAD,
};
static int bench_trace_fd = -1;
static enum bench_trace_role bench_trace_role = BENCH_ROLE_UNKNOWN;
static unsigned long long bench_trace_seq = 0;
static pid_t bench_trace_pid = -1;
static pid_t bench_trace_ppid = -1;
static const char *bench_trace_role_name(enum bench_trace_role role) {
switch (role) {
case BENCH_ROLE_PARENT:
return "parent";
case BENCH_ROLE_PID1:
return "pid1";
case BENCH_ROLE_PAYLOAD:
return "payload";
case BENCH_ROLE_UNKNOWN:
default:
return "unknown";
}
}
static long long bench_trace_now_ns(void) {
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0)
return 0;
return (long long)ts.tv_sec * 1000000000LL + (long long)ts.tv_nsec;
}
static void bench_trace_init_from_env(void) {
const char *path = getenv("SOCKPUPPET_BENCH_TRACE");
if (path == NULL || path[0] == '\0')
return;
bench_trace_fd =
open(path, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0600);
if (bench_trace_fd < 0) {
startup_warn("SOCKPUPPET_BENCH_TRACE disabled: cannot open %s: %s", path,
strerror(errno));
return;
}
}
static void bench_trace_set_role(enum bench_trace_role role) {
bench_trace_role = role;
bench_trace_pid = getpid();
bench_trace_ppid = getppid();
}
static int bench_trace_preserved_fd(void) {
return bench_trace_fd;
}
static size_t bench_json_append_char(char *buf, size_t pos, size_t cap,
char ch) {
if (pos + 1 < cap)
buf[pos++] = ch;
return pos;
}
static size_t bench_json_append_raw(char *buf, size_t pos, size_t cap,
const char *s) {
while (*s != '\0')
pos = bench_json_append_char(buf, pos, cap, *s++);
return pos;
}
static size_t bench_json_append_escaped(char *buf, size_t pos, size_t cap,
const char *s) {
static const char hex[] = "0123456789abcdef";
pos = bench_json_append_char(buf, pos, cap, '"');
if (s == NULL)
s = "";
for (const unsigned char *p = (const unsigned char *)s; *p != '\0'; ++p) {
switch (*p) {
case '"':
pos = bench_json_append_raw(buf, pos, cap, "\\\"");
break;
case '\\':
pos = bench_json_append_raw(buf, pos, cap, "\\\\");
break;
case '\b':
pos = bench_json_append_raw(buf, pos, cap, "\\b");
break;
case '\f':
pos = bench_json_append_raw(buf, pos, cap, "\\f");
break;
case '\n':
pos = bench_json_append_raw(buf, pos, cap, "\\n");
break;
case '\r':
pos = bench_json_append_raw(buf, pos, cap, "\\r");
break;
case '\t':
pos = bench_json_append_raw(buf, pos, cap, "\\t");
break;
default:
if (*p < 0x20) {
pos = bench_json_append_raw(buf, pos, cap, "\\u00");
pos = bench_json_append_char(buf, pos, cap, hex[*p >> 4]);
pos = bench_json_append_char(buf, pos, cap, hex[*p & 0xf]);
} else {
pos = bench_json_append_char(buf, pos, cap, (char)*p);
}
break;
}
}
pos = bench_json_append_char(buf, pos, cap, '"');
return pos;
}
static void bench_trace_emit(const char *event, const char *phase,
const char *status, const char *detail) {
char buf[8192];
size_t pos = 0;
int n;
long long ts_ns;
unsigned long long seq;
/* BENCH_TRACE is an opt-in startup evidence file. Its bounded records use
* the existing synchronous sink; a stalled filesystem may stall startup. */
if (bench_trace_fd < 0)
return;
ts_ns = bench_trace_now_ns();
seq = ++bench_trace_seq;
n = snprintf(buf, sizeof(buf),
"{\"ts_ns\":%lld,\"seq\":%llu,\"pid\":%ld,\"ppid\":%ld,",
ts_ns, seq, (long)bench_trace_pid, (long)bench_trace_ppid);
if (n < 0)
return;
pos = (size_t)n < sizeof(buf) ? (size_t)n : sizeof(buf) - 1;
pos = bench_json_append_raw(buf, pos, sizeof(buf), "\"role\":");
pos = bench_json_append_escaped(buf, pos, sizeof(buf),
bench_trace_role_name(bench_trace_role));
pos = bench_json_append_raw(buf, pos, sizeof(buf), ",\"event\":");
pos = bench_json_append_escaped(buf, pos, sizeof(buf), event);
pos = bench_json_append_raw(buf, pos, sizeof(buf), ",\"phase\":");
pos = bench_json_append_escaped(buf, pos, sizeof(buf), phase);
pos = bench_json_append_raw(buf, pos, sizeof(buf), ",\"status\":");
pos = bench_json_append_escaped(buf, pos, sizeof(buf), status);
pos = bench_json_append_raw(buf, pos, sizeof(buf), ",\"detail\":");
pos = bench_json_append_escaped(buf, pos, sizeof(buf), detail);
pos = bench_json_append_raw(buf, pos, sizeof(buf), "}\n");
if (pos >= sizeof(buf) - 1) {
n = snprintf(buf, sizeof(buf),
"{\"ts_ns\":%lld,\"seq\":%llu,\"pid\":%ld,\"ppid\":%ld,"
"\"role\":\"%s\",\"event\":\"trace_truncated\","
"\"phase\":\"trace\",\"status\":\"truncated\","
"\"detail\":\"record exceeded buffer\"}\n",
ts_ns, seq, (long)bench_trace_pid, (long)bench_trace_ppid,
bench_trace_role_name(bench_trace_role));
if (n < 0)
return;
pos = (size_t)n < sizeof(buf) ? (size_t)n : sizeof(buf) - 1;
}
if (write_all(bench_trace_fd, buf, pos) != (ssize_t)pos) {
int saved = errno;
close(bench_trace_fd);
bench_trace_fd = -1;
errno = saved;
}
}
static void bench_phase_begin(const char *phase) {
bench_trace_emit("phase_begin", phase, "start", NULL);
}
static void bench_phase_end(const char *phase, const char *status) {
bench_trace_emit("phase_end", phase, status, NULL);
}
static void bench_mark(const char *event, const char *phase,
const char *status, const char *detail) {
bench_trace_emit(event, phase, status, detail);
}
static int read_self_cgroup_path(char *out, size_t out_sz) {
FILE *fp = fopen("/proc/self/cgroup", "r");
char line[1024];
if (!fp)
return -1;
while (fgets(line, sizeof(line), fp) != NULL) {
if (strncmp(line, "0::", 3) != 0)
continue;
char *path = line + 3;
char *nl = strchr(path, '\n');
if (nl)
*nl = '\0';
if (*path == '\0')
path = "/";
if (snprintf(out, out_sz, "%s", path) >= (int)out_sz) {
fclose(fp);
return -1;
}
fclose(fp);
return 0;
}
fclose(fp);
return -1;
}
static int read_text_file_trimmed(const char *path, char *out, size_t out_sz) {
int fd = open(path, O_RDONLY | O_CLOEXEC);
ssize_t n;
if (fd < 0)
return -1;
if (out_sz == 0) {
close(fd);
errno = EINVAL;
return -1;
}
n = read(fd, out, out_sz - 1);
if (n < 0) {
int saved = errno;
close(fd);
errno = saved;
return -1;
}
if (close(fd) < 0)
return -1;
out[n] = '\0';
while (n > 0 && (out[n - 1] == '\n' || out[n - 1] == '\r' ||
out[n - 1] == ' ' || out[n - 1] == '\t')) {
out[--n] = '\0';
}
return 0;
}
static void report_userns_diagnostics(int err) {
char unprivileged_userns_clone[64] = "unavailable";
char max_user_namespaces[64] = "unavailable";
char apparmor_restrict[64] = "unavailable";
char lsm_stack[256] = "unavailable";
char lsm_context[256] = "unavailable";
(void)read_text_file_trimmed("/proc/sys/kernel/unprivileged_userns_clone",
unprivileged_userns_clone,
sizeof(unprivileged_userns_clone));
(void)read_text_file_trimmed("/proc/sys/user/max_user_namespaces",
max_user_namespaces,
sizeof(max_user_namespaces));
(void)read_text_file_trimmed(
"/proc/sys/kernel/apparmor_restrict_unprivileged_userns",
apparmor_restrict, sizeof(apparmor_restrict));
(void)read_text_file_trimmed("/sys/kernel/security/lsm", lsm_stack,
sizeof(lsm_stack));
(void)read_text_file_trimmed("/proc/self/attr/current", lsm_context,
sizeof(lsm_context));
fprintf(stderr,
"[sockpuppet] Error: clone(CLONE_NEWUSER|...) failed: %s\n",
strerror(err));
startup_warn("unprivileged user namespaces are required; this host did not "
"allow sandbox setup");
startup_warn("sysctl kernel.unprivileged_userns_clone=%s",
unprivileged_userns_clone);
startup_warn("sysctl user.max_user_namespaces=%s", max_user_namespaces);
startup_warn("LSM stack: %s", lsm_stack);
if (strstr(lsm_stack, "apparmor") != NULL) {
startup_warn("sysctl kernel.apparmor_restrict_unprivileged_userns=%s",
apparmor_restrict);
startup_warn("current AppArmor context: %s", lsm_context);
} else {
startup_warn("AppArmor is not active on this host; skipping AppArmor-"
"specific userns checks");
startup_warn("current LSM context: %s", lsm_context);
}
startup_warn("supported hosts need user namespaces enabled; see the docs "
"for the supported environment matrix");
}
static int cgroup_self_root(char *out, size_t out_sz) {
char rel[PATH_MAX];
if (read_self_cgroup_path(rel, sizeof(rel)) < 0)
return -1;
if (snprintf(out, out_sz, "/sys/fs/cgroup%s",
strcmp(rel, "/") == 0 ? "" : rel) >= (int)out_sz)
return -1;
return 0;
}
static const char *cgroup_delegation_state(const char *root) {
char ctrl_path[PATH_MAX];
char subtree_path[PATH_MAX];
if (root == NULL)
return "unknown";
if (snprintf(ctrl_path, sizeof(ctrl_path), "%s/cgroup.controllers", root) >=
(int)sizeof(ctrl_path) ||
snprintf(subtree_path, sizeof(subtree_path), "%s/cgroup.subtree_control",
root) >= (int)sizeof(subtree_path))
return "unknown";
if (access(ctrl_path, R_OK) < 0 || access(subtree_path, W_OK) < 0)
return "absent";
return "delegated";
}
/* Report the limit that is actually in force, not whether something could set
* one. "max" (the kernel default) and an unreadable file both mean unlimited. */
static void cgroup_effective_limit(const char *root, const char *name,
char *out, size_t out_sz) {
char path[PATH_MAX];
if (snprintf(out, out_sz, "unknown") >= (int)out_sz)
return;
if (root == NULL ||
snprintf(path, sizeof(path), "%s/%s", root, name) >= (int)sizeof(path))
return;
if (read_text_file_trimmed(path, out, out_sz) < 0)
(void)snprintf(out, out_sz, "unreadable");
}
static int parse_flow_limit_option(const char *value, int max_value,
const char *name) {
long parsed;
if (parse_long_strict(value, 1, max_value, &parsed) < 0) {
fprintf(stderr, "Invalid %s: %s (expected 1..%d)\n", name, value,
max_value);
exit(1);
}
return (int)parsed;
}
static void set_parent_death_signal(void) {
if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0)
die("PR_SET_PDEATHSIG");
}
/* Fork-style clone: the child gets a private copy of memory (its stack
* included) and of the file table, returns 0 like fork(), and is reaped with
* SIGCHLD. glibc and musl do not refresh their cached thread ID in a child
* created this way; PID 1 never calls raise(), abort() or pthread_kill(),
* which are the functions that read it. Every supported architecture takes
* (flags, stack, parent_tid, child_tid or tls, ...); a NULL stack means the
* child continues on its copy of the current one. */
static pid_t sp_clone_namespace_init(unsigned long namespace_flags) {
return (pid_t)syscall(SYS_clone, namespace_flags | SIGCHLD, 0UL, 0UL, 0UL,
0UL);
}
static void arm_parent_death_signal(pid_t expected_parent) {
set_parent_death_signal();
if (getppid() != expected_parent)
_exit(1);
}
/* PID 1 cannot use arm_parent_death_signal(): its parent (the broker) lives
* outside the new PID namespace, so getppid() returns 0 whether or not the
* broker is alive. Instead the broker keeps the write end of a liveness pipe
* open for its lifetime; POLLHUP on the read end after arming
* PR_SET_PDEATHSIG means the broker died inside the clone-to-arm window and
* the death signal will never fire. */
static void arm_pid1_parent_death_signal(int liveness_rd) {
set_parent_death_signal();
struct pollfd pfd = {.fd = liveness_rd, .events = 0, .revents = 0};
int rc;
do {
rc = poll(&pfd, 1, 0);
} while (rc < 0 && errno == EINTR);
if (rc < 0)
die("poll pid1 parent liveness");
if (rc > 0) {
fprintf(stderr, "[sockpuppet] Error: broker died before "
"PID 1 armed parent-death signal\n");
_exit(1);
}
close(liveness_rd);
}
static void mkdir_if_missing(const char *path, mode_t mode) {
if (mkdir(path, mode) < 0 && errno != EEXIST)
die(path);
}
#include <linux/landlock.h>
#define SP_HAVE_LANDLOCK 1
struct fs_sandbox {
char resolved_cwd[PATH_MAX];
/* Canonical host path of the staging tmpfs. The sandbox tree is assembled
* under it in the host view, then a single pivot_root into final_root
* detaches both the host root and the staging tree (M7, 2026-10-03). */
char staging[PATH_MAX];
char final_root[PATH_MAX];
int staging_root_fd;
struct sp_mount_scope scopes[3];
size_t scope_count;
int cwd_readonly_fallback;
int readonly_binds_enforced;
/* --rootfs without a setup script: add a generated /etc/resolv.conf when
* the image has none (a setup script may want to write its own). */
int provide_rootfs_resolver;
};
/* out = sandbox->staging + name, where name starts with '/'. */
static void sp_staging_path(const struct fs_sandbox *sandbox, const char *name,
char *out, size_t size) {
if (snprintf(out, size, "%s%s", sandbox->staging, name) >= (int)size)
die("staging path");
}
static void sp_mount_scope_init(struct sp_mount_scope *scope) {
memset(scope, 0, sizeof(*scope));
scope->source_fd = -1;
scope->target_fd = -1;
scope->upper_fd = -1;
scope->work_fd = -1;
}
static void sp_fs_sandbox_init(struct fs_sandbox *sandbox) {
memset(sandbox, 0, sizeof(*sandbox));
sandbox->staging_root_fd = -1;
for (size_t i = 0; i < sizeof(sandbox->scopes) / sizeof(sandbox->scopes[0]);
++i)
sp_mount_scope_init(&sandbox->scopes[i]);
}
static void sp_close_fs_scopes(struct fs_sandbox *sandbox) {
sp_close_owned_fd_nointr(&sandbox->staging_root_fd);
for (size_t i = 0; i < sandbox->scope_count; ++i) {
struct sp_mount_scope *scope = &sandbox->scopes[i];
sp_close_owned_fd_nointr(&scope->source_fd);
sp_close_owned_fd_nointr(&scope->target_fd);
sp_close_owned_fd_nointr(&scope->upper_fd);
sp_close_owned_fd_nointr(&scope->work_fd);
}
}
static enum sp_upper_mode sp_view_upper_mode(enum sp_view_mode mode) {
if (mode == SP_VIEW_COW_DISPOSABLE)
return SP_UPPER_DISPOSABLE;
if (mode == SP_VIEW_COW_RETAINED)
return SP_UPPER_RETAINED;
return SP_UPPER_NONE;
}
static int sp_add_fs_scope(struct fs_sandbox *sandbox,
enum sp_scope_kind kind,
const struct sp_bind_spec *bind,
enum sp_bind_mode mode,
enum sp_upper_mode upper_mode,
const char *source, char *error,
size_t error_size) {
struct sp_mount_scope *scope = &sandbox->scopes[sandbox->scope_count++];
scope->bind = bind;
scope->kind = kind;
scope->mode = mode;
scope->upper_mode = upper_mode;
scope->source_fd = sp_open_directory_nosymlinks(source);
if (scope->source_fd < 0) {
snprintf(error, error_size, "open %s source",
kind == SP_SCOPE_ROOTFS ? "rootfs"
: kind == SP_SCOPE_BIND ? "bind"
: "workspace");
return -1;
}
return 0;
}
static int sp_open_scope_storage(int storage_fd, const char *name,
struct sp_mount_scope *scope, char *error,
size_t error_size) {
char upper[64];
char work[64];
struct stat source_stat;
int upper_chmod_fd = -1;
if (snprintf(upper, sizeof(upper), "/%s-upper", name) >=
(int)sizeof(upper) ||
snprintf(work, sizeof(work), "/%s-work", name) >= (int)sizeof(work)) {
errno = ENAMETOOLONG;
snprintf(error, error_size, "scope storage name too long");
return -1;
}
if ((mkdirat(storage_fd, upper + 1, 0700) < 0 && errno != EEXIST) ||
(mkdirat(storage_fd, work + 1, 0700) < 0 && errno != EEXIST)) {
snprintf(error, error_size, "create %s overlay storage", name);
return -1;
}
scope->upper_fd = sp_open_directory_beneath(storage_fd, upper);
if (scope->upper_fd < 0) {
snprintf(error, error_size, "open %s upper directory", name);
return -1;
}
/* Overlayfs exposes the upper directory inode as the merged root. Keep the
* backing storage private through its mode-0700 parent, but make the visible
* overlay root honor the source root's traversal policy. */
if (fstat(scope->source_fd, &source_stat) < 0 ||
(upper_chmod_fd = openat(scope->upper_fd, ".",
O_RDONLY | O_DIRECTORY | O_CLOEXEC)) < 0 ||
fchmod(upper_chmod_fd, source_stat.st_mode & 07777) < 0) {
int saved = errno;
sp_close_owned_fd_nointr(&upper_chmod_fd);
errno = saved;
snprintf(error, error_size, "set %s overlay root mode", name);
return -1;
}
sp_close_owned_fd_nointr(&upper_chmod_fd);
scope->work_fd = sp_open_directory_beneath(storage_fd, work);
if (scope->work_fd < 0) {
snprintf(error, error_size, "open %s work directory", name);
return -1;
}
return 0;
}
static int sp_acquire_fs_scopes(struct fs_sandbox *sandbox,
const struct sp_fs_config *config,
const char *cwd, const char *scratch,
int approved_retained_fd,
char *error, size_t error_size) {
int disposable_fd = -1;
int retained_fd = -1;
int need_retained = 0;
if (config->rootfs_path != NULL &&
sp_add_fs_scope(sandbox, SP_SCOPE_ROOTFS, NULL,
config->rootfs_view == SP_VIEW_READONLY ? SP_BIND_RO
: SP_BIND_COW,
sp_view_upper_mode(config->rootfs_view),
config->rootfs_path, error, error_size) < 0)
goto fail;
if (sp_add_fs_scope(
sandbox, SP_SCOPE_WORKSPACE, NULL,
config->workspace_view == SP_VIEW_READONLY ? SP_BIND_RO
: SP_BIND_COW,
sp_view_upper_mode(config->workspace_view), cwd, error,
error_size) < 0)
goto fail;
if (config->bind_enabled &&
sp_add_fs_scope(sandbox, SP_SCOPE_BIND, &config->bind,
config->bind.mode,
config->bind.mode == SP_BIND_COW ? SP_UPPER_DISPOSABLE
: SP_UPPER_NONE,
config->bind.source, error, error_size) < 0)
goto fail;
for (size_t i = 0; i < sandbox->scope_count; ++i) {
need_retained |= sandbox->scopes[i].upper_mode == SP_UPPER_RETAINED;
}
/* The namespace child made every mount private right after unshare(). */
if (mount("tmpfs", scratch, "tmpfs", MS_NODEV | MS_NOSUID,
"mode=0700,size=128m") < 0) {
snprintf(error, error_size, "mount filesystem staging storage");
goto fail;
}
disposable_fd = sp_open_directory_nosymlinks(scratch);
if (disposable_fd < 0) {
snprintf(error, error_size, "open filesystem staging storage");
goto fail;
}
if (need_retained) {
struct stat approved;
struct stat reopened;
retained_fd =
sp_open_directory_nosymlinks(config->retained_directory);
if (retained_fd < 0) {
snprintf(error, error_size, "open retained overlay storage");
goto fail;
}
if (fstat(approved_retained_fd, &approved) < 0 ||
fstat(retained_fd, &reopened) < 0) {
snprintf(error, error_size, "retained directory changed after locking");
goto fail;
}
if (approved.st_dev != reopened.st_dev || approved.st_ino != reopened.st_ino) {
errno = ESTALE;
snprintf(error, error_size, "retained directory changed after locking");
goto fail;
}
}
for (size_t i = 0; i < sandbox->scope_count; ++i) {
struct sp_mount_scope *scope = &sandbox->scopes[i];
char name[48];
int storage_fd;
if (scope->upper_mode == SP_UPPER_NONE)
continue;
if (scope->kind == SP_SCOPE_ROOTFS)
snprintf(name, sizeof(name), "rootfs");
else if (scope->kind == SP_SCOPE_WORKSPACE)
snprintf(name, sizeof(name), "workspace");
else
snprintf(name, sizeof(name), "bind-%s", scope->bind->name);
storage_fd = scope->upper_mode == SP_UPPER_RETAINED ? retained_fd
: disposable_fd;
if (sp_open_scope_storage(storage_fd, name, scope, error, error_size) < 0)
goto fail;
}
sp_close_owned_fd_nointr(&disposable_fd);
sp_close_owned_fd_nointr(&retained_fd);
return 0;
fail: {
int saved = errno;
sp_close_owned_fd_nointr(&disposable_fd);
sp_close_owned_fd_nointr(&retained_fd);
sp_close_fs_scopes(sandbox);
errno = saved;
return -1;
}
}
static void write_text_file(const char *path, const char *data) {
int fd = open(path, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644);
if (fd < 0)
die(path);
size_t len = strlen(data);
if (write(fd, data, len) != (ssize_t)len)
die("write");
close(fd);
}
static int path_exists(const char *path) {
struct stat st;
return stat(path, &st) == 0;
}
static void mkdir_parents(const char *path, mode_t mode) {
char tmp[PATH_MAX];
size_t len = strlen(path);
if (len >= sizeof(tmp))
die("mkdir_parents");
memcpy(tmp, path, len + 1);
for (char *p = tmp + 1; *p; ++p) {
if (*p != '/')
continue;
*p = '\0';
mkdir_if_missing(tmp, mode);
*p = '/';
}
mkdir_if_missing(tmp, mode);
}
static int path_contains(const char *base, const char *path) {
size_t len = strlen(base);
if (strcmp(base, "/") == 0)
return 1;
if (strncmp(base, path, len) != 0)
return 0;
return path[len] == '\0' || path[len] == '/';
}
static void path_append(char *dst, size_t dst_size, const char *root,
const char *suffix) {
if (snprintf(dst, dst_size, "%s%s", root, suffix) >= (int)dst_size)
die("path too long");
}
static void normalize_absolute_path(const char *path, char *out,
size_t out_size) {
const char *segments[PATH_MAX / 2];
size_t count = 0;
char tmp[PATH_MAX];
char *save = NULL;
char *tok;
size_t len = strlen(path);
if (len >= sizeof(tmp))
die("normalize path");
memcpy(tmp, path, len + 1);
for (tok = strtok_r(tmp, "/", &save); tok; tok = strtok_r(NULL, "/", &save)) {
if (strcmp(tok, ".") == 0 || *tok == '\0')
continue;
if (strcmp(tok, "..") == 0) {
if (count > 0)
--count;
continue;
}
segments[count++] = tok;
}
if (snprintf(out, out_size, "/") >= (int)out_size)
die("normalize path");
for (size_t i = 0; i < count; ++i) {
size_t used = strlen(out);
if (snprintf(out + used, out_size - used, "%s%s", i == 0 ? "" : "/",
segments[i]) >= (int)(out_size - used))
die("normalize path");
}
}
/* Clone dirfd/path (flags: AT_EMPTY_PATH for a descriptor, AT_RECURSIVE for
* submounts), set attr on the detached clone and attach it at target_fd. The
* attributes are recursive exactly when the clone is, so every cloned submount
* is protected before it becomes visible. Returns 0, or -1 with errno set. */
static int sp_attach_clone(int dirfd, const char *path, unsigned int flags,
const struct mount_attr *attr, int target_fd) {
int tree_fd = (int)syscall(SYS_open_tree, dirfd, path,
OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | flags);
int rc = tree_fd < 0 ||
syscall(SYS_mount_setattr, tree_fd, "",
AT_EMPTY_PATH | (flags & AT_RECURSIVE), attr,
sizeof(*attr)) < 0 ||
syscall(SYS_move_mount, tree_fd, "", target_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0
? -1 : 0;
int saved = errno;
if (tree_fd >= 0)
close(tree_fd);
errno = saved;
return rc;
}
/* The caller has already checked that src exists; a source that disappears
* since still fails below. Protect all cloned submounts before attachment;
* MS_REC on a legacy bind-remount only changes the top mount. */
static void bind_mount_dir_readonly(const char *src, const char *dst) {
struct mount_attr attr = {
.attr_set = MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID,
};
mkdir_parents(dst, 0755);
int target_fd = sp_open_directory_nosymlinks(dst);
if (target_fd < 0 ||
sp_attach_clone(AT_FDCWD, src, AT_RECURSIVE, &attr, target_fd) < 0)
die(src);
close(target_fd);
}
/* dst's parent directory must already exist. */
static void bind_mount_file_in_dir(const char *src, const char *dst,
int nosuid) {
if (!path_exists(src))
return;
if (!path_exists(dst)) {
int fd = open(dst, O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0644);
if (fd < 0)
die(dst);
close(fd);
}
if (mount(src, dst, NULL, MS_BIND, NULL) < 0)
die(src);
if (nosuid &&
mount(NULL, dst, NULL, MS_BIND | MS_REMOUNT | MS_NOSUID, NULL) < 0) {
if (errno == EPERM || errno == EINVAL || errno == EACCES) {
DBG("nosuid remount skipped for %s (%s)", dst, strerror(errno));
} else {
die(dst);
}
}
}
/* Mountpoints directly beneath the final root. That root can be a
* caller-supplied rootfs or an overlay of the host root, so a component may be
* a symlink; mount(2) would follow it out of the final root. Create the
* directory if missing, then re-open it with RESOLVE_BENEATH|NO_SYMLINKS
* (mkdirat returns EEXIST for an existing symlink too) and mount onto the fd. */
static int sp_open_final_mountpoint(const struct fs_sandbox *sandbox,
const char *path, mode_t mode) {
int fd;
if (mkdirat(sandbox->staging_root_fd, path + 1, mode) < 0 && errno != EEXIST)
die(path);
fd = sp_open_directory_beneath(sandbox->staging_root_fd, path);
if (fd < 0)
die(path);
return fd;
}
/* options holds key/value pairs, NULL-terminated. */
static void sp_mount_new_fs_at(int target_fd, const char *fstype,
const char *const *options,
unsigned int attrs, const char *what) {
int fs_fd = (int)syscall(SYS_fsopen, fstype, FSOPEN_CLOEXEC);
int mnt_fd = -1;
if (fs_fd < 0)
die(what);
for (size_t i = 0; options[i] != NULL; i += 2) {
if (syscall(SYS_fsconfig, fs_fd, FSCONFIG_SET_STRING, options[i],
options[i + 1], 0) < 0)
die(what);
}
if (syscall(SYS_fsconfig, fs_fd, FSCONFIG_CMD_CREATE, NULL, NULL, 0) < 0)
die(what);
mnt_fd = (int)syscall(SYS_fsmount, fs_fd, FSMOUNT_CLOEXEC, attrs);
if (mnt_fd < 0)
die(what);
if (syscall(SYS_move_mount, mnt_fd, "", target_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0)
die(what);
close(mnt_fd);
close(fs_fd);
}
static int move_staging_proc(const struct fs_sandbox *sandbox) {
char staging_proc[PATH_MAX];
sp_staging_path(sandbox, "/proc", staging_proc, sizeof(staging_proc));
int target_fd = sp_open_final_mountpoint(sandbox, "/proc", 0555);
int tree_fd = (int)syscall(SYS_open_tree, AT_FDCWD, staging_proc,
OPEN_TREE_CLOEXEC);
if (tree_fd < 0 ||
syscall(SYS_move_mount, tree_fd, "", target_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0)
die("mount proc");
close(target_fd);
return tree_fd;
}
static void mount_private_tmp(const struct fs_sandbox *sandbox) {
static const char *const options[] = {"mode", "1777", "size", "64m", NULL};
int target_fd = sp_open_final_mountpoint(sandbox, "/tmp", 01777);
sp_mount_new_fs_at(target_fd, "tmpfs", options,
MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV, "mount tmpfs /tmp");
close(target_fd);
}
static void mount_minimal_dev(const struct fs_sandbox *sandbox) {
/* /dev/random is a symlink to urandom below: since Linux 5.6 they differ
* only before the CRNG is initialized, long before a sandbox starts. Each
* bind is a mount on the startup path; /dev/full is not provided. */
static const char *const dev_files[] = {
"/dev/null", "/dev/zero", "/dev/urandom",
};
struct mount_attr attr = {.attr_set = MOUNT_ATTR_NOSUID};
char dev_min[PATH_MAX];
sp_staging_path(sandbox, "/dev-min", dev_min, sizeof(dev_min));
int target_fd = sp_open_final_mountpoint(sandbox, "/dev", 0755);
int tree_fd = (int)syscall(SYS_open_tree, AT_FDCWD, dev_min,
OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
char dev_root[64], path[PATH_MAX];
if (tree_fd < 0)
die("mount /dev");
if (syscall(SYS_move_mount, tree_fd, "", target_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0)
die("mount /dev");
close(target_fd);
snprintf(dev_root, sizeof(dev_root), "/proc/self/fd/%d", tree_fd);
/* Populate only the attached view; /dev-min has no child mounts to clone.
* Sources are the host's device nodes. */
for (size_t i = 0; i < sizeof(dev_files) / sizeof(dev_files[0]); ++i) {
char dst[PATH_MAX];
path_append(dst, sizeof(dst), dev_root, dev_files[i] + 4);
bind_mount_file_in_dir(dev_files[i], dst, 0);
}
/* /dev/tty resolves to the caller's controlling terminal. Only project it
* for interactive sessions, where the payload's ctty is the sandbox PTY.
* A noninteractive payload has no ctty and must not be able to reach the
* launcher's terminal through this node. */
if (interactive_stdio) {
path_append(path, sizeof(path), dev_root, "/tty");
bind_mount_file_in_dir("/dev/tty", path, 0);
}
path_append(path, sizeof(path), dev_root, "/pts");
mkdir_if_missing(path, 0755);
if (mount("devpts", path, "devpts",
MS_NOSUID | MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620") < 0)
die("mount devpts");
unlinkat(tree_fd, "ptmx", 0);
if (symlinkat("pts/ptmx", tree_fd, "ptmx") < 0)
die("symlink /dev/ptmx");
if (symlinkat("urandom", tree_fd, "random") < 0)
die("symlink /dev/random");
if (syscall(SYS_mount_setattr, tree_fd, "", AT_EMPTY_PATH | AT_RECURSIVE,
&attr, sizeof(attr)) < 0)
die("mount /dev nosuid");
close(tree_fd);
}
static int sp_try_runtime_alias(const struct fs_sandbox *sandbox,
const char *path, const char *src,
const char *alias) {
char link[PATH_MAX];
struct stat source, target;
if (path_contains(sandbox->resolved_cwd, "/usr") ||
sp_paths_overlap(sandbox->resolved_cwd, path) ||
sp_paths_overlap(sandbox->resolved_cwd, alias))
return 0;
for (size_t i = 0; i < sandbox->scope_count; ++i) {
const struct sp_mount_scope *scope = &sandbox->scopes[i];
if (scope->kind == SP_SCOPE_BIND &&
(sp_paths_overlap(scope->bind->target, path) ||
sp_paths_overlap(scope->bind->target, alias)))
return 0;
}
ssize_t length = readlink(src, link, sizeof(link) - 1);
if (length < 0)
return 0;
link[length] = '\0';
if (strcmp(link, alias) != 0 && strcmp(link, alias + 1) != 0)
return 0;
int fd = sp_open_directory_beneath(sandbox->staging_root_fd, alias);
if (fd < 0)
return 0;
int matches = stat(src, &source) == 0 && fstat(fd, &target) == 0 &&
source.st_dev == target.st_dev && source.st_ino == target.st_ino;
close(fd);
if (!matches)
return 0;
if (symlinkat(alias + 1, sandbox->staging_root_fd, path + 1) < 0)
die(path);
return 1;
}
static void mount_runtime_tree(struct fs_sandbox *sandbox) {
static const struct {
const char *path;
const char *alias;
} runtime_dirs[] = {
{"/usr", NULL}, {"/bin", "/usr/bin"},
{"/sbin", "/usr/sbin"}, {"/lib", "/usr/lib"},
{"/lib64", "/usr/lib64"}, {"/etc", NULL}, {"/nix", NULL},
};
for (size_t i = 0; i < sizeof(runtime_dirs) / sizeof(runtime_dirs[0]); ++i) {
char src[PATH_MAX];
char dst[PATH_MAX];
const char *path = runtime_dirs[i].path;
if (path_contains(sandbox->resolved_cwd, path))
continue;
path_append(src, sizeof(src), "", path);
path_append(dst, sizeof(dst), sandbox->final_root, path);
if (!path_exists(src))
continue;
if (runtime_dirs[i].alias != NULL &&
sp_try_runtime_alias(sandbox, path, src, runtime_dirs[i].alias))
continue;
bind_mount_dir_readonly(src, dst);
}
}
static const char *generated_resolv_conf(void) {
return egress_mode == EGRESS_SOCKS && socks_proxy.remote_dns
? "nameserver 10.0.0.1\n"
: "nameserver 8.8.8.8\n";
}
/* --rootfs: give an image without /etc/resolv.conf the generated resolver.
* An existing entry (file or symlink) is the image's choice and is left alone.
* The file is only created in a disposable overlay: a read-only rootfs cannot
* take it, and a retained upper would keep a stale copy for later launches. */
static void provide_rootfs_resolv_conf(const struct fs_sandbox *sandbox,
int disposable_upper) {
struct open_how probe = {
.flags = O_PATH | O_NOFOLLOW | O_CLOEXEC,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS,
};
struct open_how create = {
.flags = O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC,
.mode = 0644,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS,
};
const char *text = generated_resolv_conf();
size_t len = strlen(text);
int fd = (int)syscall(SYS_openat2, sandbox->staging_root_fd,
"etc/resolv.conf", &probe, sizeof(probe));
if (fd >= 0) {
close(fd);
return;
}
if (errno != ENOENT) {
startup_warn("rootfs /etc/resolv.conf not inspected (%s); DNS is left "
"to the image", strerror(errno));
return;
}
if (!disposable_upper) {
startup_warn("rootfs has no /etc/resolv.conf and its root is not a "
"disposable overlay; DNS is not configured (use "
"--overlay=tmpfs to provide one)");
return;
}
/* Creating under a lower /etc copies it up, which fails (EACCES, EOVERFLOW)
* when the image's ids are not mapped here; such an image launched before
* this feature existed, so treat it like a read-only root. */
if ((mkdirat(sandbox->staging_root_fd, "etc", 0755) < 0 && errno != EEXIST) ||
(fd = (int)syscall(SYS_openat2, sandbox->staging_root_fd,
"etc/resolv.conf", &create, sizeof(create))) < 0) {
startup_warn("cannot create rootfs /etc/resolv.conf (%s); DNS is not "
"configured", strerror(errno));
return;
}
if (write(fd, text, len) != (ssize_t)len)
die("write rootfs /etc/resolv.conf");
close(fd);
}
/* Bind the generated resolver over the final root's /etc/resolv.conf, or
* over the target of its symlink. The link is resolved inside the final root
* (an absolute target is taken relative to it, and ".." cannot climb out),
* and every component is created or opened beneath the root without
* following symlinks, so nothing is created or mounted outside the sandbox
* tree, which is still assembled in the host view. A target that cannot be
* reached that way leaves DNS unconfigured with a warning. */
static void mount_resolv_conf(const struct fs_sandbox *sandbox) {
struct mount_attr attr = {.attr_set = MOUNT_ATTR_NOSUID};
struct open_how dir_how = {
.flags = O_PATH | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS,
};
struct open_how file_how = {
.flags = O_WRONLY | O_CREAT | O_NOFOLLOW | O_CLOEXEC,
.mode = 0644,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS,
};
char tmp_path[PATH_MAX], link[PATH_MAX], virt[PATH_MAX], joined[PATH_MAX];
int dir_fd, file_fd;
ssize_t len;
sp_staging_path(sandbox, "/resolv.conf.tmp", tmp_path, sizeof(tmp_path));
write_text_file(tmp_path, generated_resolv_conf());
len = readlinkat(sandbox->staging_root_fd, "etc/resolv.conf", link,
sizeof(link) - 1);
if (len < 0) {
memcpy(virt, "/etc/resolv.conf", sizeof("/etc/resolv.conf"));
} else {
link[len] = '\0';
if (snprintf(joined, sizeof(joined), "%s%s", link[0] == '/' ? "" : "/etc/",
link) >= (int)sizeof(joined))
die("resolv.conf link");
normalize_absolute_path(joined, virt, sizeof(virt));
}
char *name = strrchr(virt, '/');
if (name == NULL || name[1] == '\0')
die("resolv.conf target");
*name++ = '\0';
dir_fd = (int)syscall(SYS_openat2, sandbox->staging_root_fd, ".", &dir_how,
sizeof(dir_how));
for (char *save = NULL, *part = strtok_r(virt, "/", &save);
dir_fd >= 0 && part != NULL; part = strtok_r(NULL, "/", &save)) {
int next;
if (mkdirat(dir_fd, part, 0755) < 0 && errno != EEXIST) {
close(dir_fd);
dir_fd = -1;
break;
}
next = (int)syscall(SYS_openat2, dir_fd, part, &dir_how, sizeof(dir_how));
close(dir_fd);
dir_fd = next;
}
file_fd = dir_fd < 0 ? -1
: (int)syscall(SYS_openat2, dir_fd, name, &file_how,
sizeof(file_how));
if (file_fd < 0 && dir_fd >= 0 && (errno == EROFS || errno == EACCES)) {
struct open_how existing = dir_how;
existing.flags = O_PATH | O_NOFOLLOW | O_CLOEXEC;
file_fd = (int)syscall(SYS_openat2, dir_fd, name, &existing,
sizeof(existing));
}
if (dir_fd >= 0)
close(dir_fd);
if (file_fd < 0) {
startup_warn("cannot place /etc/resolv.conf in the sandbox (%s); DNS is "
"not configured", strerror(errno));
return;
}
if (sp_attach_clone(AT_FDCWD, tmp_path, 0, &attr, file_fd) < 0)
die("mount resolv.conf");
close(file_fd);
}
static int overlay_fallback_allowed_errno(int err) {
return err == EINVAL || err == ENODEV || err == EOPNOTSUPP ||
err == EPERM || err == EACCES;
}
static void die_writable_cwd_overlay_required(const struct fs_sandbox *sandbox,
int err) {
fprintf(stderr,
"Writable cwd overlay required but overlayfs is unavailable for %s "
"(%s)\n",
sandbox->resolved_cwd, strerror(err));
exit(1);
}
static struct sp_mount_scope *sp_find_fs_scope(struct fs_sandbox *sandbox,
enum sp_scope_kind kind) {
for (size_t i = 0; i < sandbox->scope_count; ++i) {
if (sandbox->scopes[i].kind == kind)
return &sandbox->scopes[i];
}
return NULL;
}
static int sp_attach_detached(struct sp_mount_scope *scope) {
struct mount_attr attr = {
.attr_set = MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV |
(scope->mode == SP_BIND_RO ? MOUNT_ATTR_RDONLY : 0),
};
if (sp_attach_clone(scope->source_fd, "", AT_EMPTY_PATH, &attr,
scope->target_fd) < 0) {
int saved = errno;
DBG("detached scope attach failed: %s", strerror(saved));
errno = saved;
return -1;
}
scope->attached = 1;
return 0;
}
static int sp_attach_readonly_fallback(const struct fs_sandbox *sandbox,
struct sp_mount_scope *scope,
const char *target) {
struct mount_attr attr = {
.attr_set = MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID,
};
char source[64];
if (snprintf(source, sizeof(source), "/proc/self/fd/%d",
scope->source_fd) >= (int)sizeof(source)) {
errno = ENAMETOOLONG;
return -1;
}
if (mount(source, target, NULL, MS_BIND | MS_REC, NULL) < 0)
return -1;
/* The staging tmpfs is mounted in the host view while the tree is built.
* If the workspace is its parent directory (cwd /tmp), the recursive bind
* copied it: detach that copy so no staging content is visible. */
{
const char *leaf = strrchr(sandbox->staging, '/');
size_t parent_len = leaf == NULL ? 0 : (size_t)(leaf - sandbox->staging);
char copy[PATH_MAX];
if (leaf != NULL && strlen(sandbox->resolved_cwd) == parent_len &&
strncmp(sandbox->resolved_cwd, sandbox->staging, parent_len) == 0) {
if (snprintf(copy, sizeof(copy), "%s%s", target, leaf) >=
(int)sizeof(copy) ||
umount2(copy, MNT_DETACH) < 0) {
int saved = errno;
(void)umount2(target, MNT_DETACH);
errno = saved;
return -1;
}
}
}
int tree_fd = (int)syscall(SYS_open_tree, AT_FDCWD, target,
OPEN_TREE_CLOEXEC);
int rc = tree_fd < 0 ? -1 :
(int)syscall(SYS_mount_setattr, tree_fd, "", AT_EMPTY_PATH | AT_RECURSIVE,
&attr, sizeof(attr));
int saved = errno;
if (tree_fd >= 0)
close(tree_fd);
if (rc < 0) {
(void)umount2(target, MNT_DETACH);
errno = saved;
return -1;
}
scope->attached = 1;
return 1;
}
static int sp_attach_overlay(const struct fs_sandbox *sandbox,
struct sp_mount_scope *scope) {
char options[256];
char staging[PATH_MAX];
int tree_fd;
sp_staging_path(sandbox,
scope->kind == SP_SCOPE_ROOTFS ? "/overlay-rootfs"
: scope->kind == SP_SCOPE_WORKSPACE ? "/overlay-workspace"
: "/overlay-bind",
staging, sizeof(staging));
if (snprintf(options, sizeof(options),
"lowerdir=/proc/self/fd/%d,upperdir=/proc/self/fd/%d,"
"workdir=/proc/self/fd/%d,userxattr",
scope->source_fd, scope->upper_fd, scope->work_fd) >=
(int)sizeof(options)) {
errno = ENAMETOOLONG;
return -1;
}
mkdir_parents(staging, 0755);
/* MS_NOSUID | MS_NODEV here are the mount's final attributes; the mount is
* then moved, not cloned, so they carry over unchanged. */
if (mount("overlay", staging, "overlay", MS_NOSUID | MS_NODEV, options) < 0) {
DBG("mount overlay scope failed: %s", strerror(errno));
return -1;
}
tree_fd = (int)syscall(SYS_open_tree, AT_FDCWD, staging, OPEN_TREE_CLOEXEC);
if (tree_fd < 0) {
DBG("open_tree overlay failed: %s", strerror(errno));
return -1;
}
if (syscall(SYS_move_mount, tree_fd, "", scope->target_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0) {
int saved = errno;
DBG("move_mount overlay failed: %s", strerror(saved));
close(tree_fd);
errno = saved;
return -1;
}
close(tree_fd);
scope->attached = 1;
return 0;
}
static int sp_attach_fs_scope(struct fs_sandbox *sandbox,
struct sp_mount_scope *scope,
const char *target_path,
int allow_readonly_fallback) {
scope->target_fd = scope->kind == SP_SCOPE_ROOTFS
? sp_open_directory_nosymlinks(target_path)
: sp_open_directory_beneath(sandbox->staging_root_fd,
target_path + strlen(sandbox->final_root));
if (scope->target_fd < 0)
return -1;
if (scope->upper_mode != SP_UPPER_NONE &&
sp_attach_overlay(sandbox, scope) == 0)
return 0;
if (scope->upper_mode == SP_UPPER_NONE)
return sp_attach_detached(scope);
int overlay_errno = errno;
if (!allow_readonly_fallback ||
!overlay_fallback_allowed_errno(overlay_errno)) {
errno = overlay_errno;
return -1;
}
if (require_writable_cwd)
die_writable_cwd_overlay_required(sandbox, overlay_errno);
fprintf(stderr,
"[sockpuppet] Warning: cwd overlay unavailable for %s (%s); using "
"read-only bind fallback\n",
sandbox->resolved_cwd, strerror(overlay_errno));
scope->mode = SP_BIND_RO;
if (sp_attach_detached(scope) < 0) {
int readonly = sp_attach_readonly_fallback(sandbox, scope, target_path);
if (readonly < 0)
return -1;
sandbox->readonly_binds_enforced = readonly;
}
sandbox->cwd_readonly_fallback = 1;
return 0;
}
static void setup_final_root(struct fs_sandbox *sandbox,
int allow_workspace_fallback) {
struct sp_mount_scope *rootfs =
sp_find_fs_scope(sandbox, SP_SCOPE_ROOTFS);
struct sp_mount_scope *workspace =
sp_find_fs_scope(sandbox, SP_SCOPE_WORKSPACE);
char target[PATH_MAX];
if (rootfs != NULL) {
sp_staging_path(sandbox, "/rootfs", sandbox->final_root,
sizeof(sandbox->final_root));
if (sp_attach_fs_scope(sandbox, rootfs, sandbox->final_root, 0) < 0)
die("attach rootfs");
sandbox->staging_root_fd =
sp_open_directory_nosymlinks(sandbox->final_root);
if (sandbox->staging_root_fd < 0)
die("open staged rootfs");
path_append(target, sizeof(target), sandbox->final_root, "/workspace");
if (mkdirat(sandbox->staging_root_fd, "workspace", 0755) < 0 &&
errno != EEXIST)
die("mkdir workspace");
if (sp_attach_fs_scope(sandbox, workspace, target, 0) < 0)
die("attach workspace");
memcpy(sandbox->resolved_cwd, "/workspace", sizeof("/workspace"));
mount_private_tmp(sandbox);
} else {
/* prepare_fs_sandbox() refuses cwd "/", so the workspace is always a
* subtree placed at its own path inside the synthetic /sandbox. */
sp_staging_path(sandbox, "/sandbox", sandbox->final_root,
sizeof(sandbox->final_root));
if (mount(sandbox->final_root, sandbox->final_root, NULL, MS_BIND, NULL) < 0)
die("mount final host root");
sandbox->staging_root_fd =
sp_open_directory_nosymlinks(sandbox->final_root);
if (sandbox->staging_root_fd < 0)
die("open staged root");
mount_runtime_tree(sandbox);
if (!path_contains(sandbox->resolved_cwd, "/tmp"))
mount_private_tmp(sandbox);
path_append(target, sizeof(target), sandbox->final_root,
sandbox->resolved_cwd);
mkdir_parents(target, 0755);
if (sp_attach_fs_scope(sandbox, workspace, target,
allow_workspace_fallback) < 0)
die("attach cwd");
}
struct sp_mount_scope *bind = sp_find_fs_scope(sandbox, SP_SCOPE_BIND);
if (bind != NULL) {
if (rootfs == NULL &&
sp_paths_overlap(bind->bind->target, sandbox->resolved_cwd)) {
errno = EINVAL;
die("bind target overlaps workspace");
}
path_append(target, sizeof(target), sandbox->final_root, bind->bind->target);
if (sp_attach_fs_scope(sandbox, bind, target, 0) < 0)
die("attach bind volume");
}
mount_minimal_dev(sandbox);
if (rootfs == NULL)
mount_resolv_conf(sandbox);
else if (sandbox->provide_rootfs_resolver)
provide_rootfs_resolv_conf(sandbox,
rootfs->upper_mode == SP_UPPER_DISPOSABLE);
}
static int sp_copy_sealed_script(int source, int proc_fd) {
struct stat before, after;
char buffer[4096];
size_t total = 0;
int sealed = -1;
int readonly = -1;
const int seals = F_SEAL_WRITE | F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL;
if (fstat(source, &before) < 0)
return -1;
if (!S_ISREG(before.st_mode)) {
errno = EINVAL;
return -1;
}
if (before.st_size < 0 || before.st_size > SP_SETUP_SCRIPT_MAX) {
errno = EFBIG;
return -1;
}
sealed = memfd_create("sockpuppet-setup", MFD_CLOEXEC | MFD_ALLOW_SEALING);
if (sealed < 0)
return -1;
for (;;) {
ssize_t count = read(source, buffer, sizeof(buffer));
if (count < 0) {
if (errno == EINTR)
continue;
goto out;
}
if (count == 0)
break;
total += (size_t)count;
if (total > SP_SETUP_SCRIPT_MAX) {
errno = EFBIG;
goto out;
}
if (write_all(sealed, buffer, (size_t)count) != count)
goto out;
}
if (fstat(source, &after) < 0)
goto out;
if ((off_t)total != before.st_size || before.st_size != after.st_size ||
before.st_mtim.tv_sec != after.st_mtim.tv_sec ||
before.st_mtim.tv_nsec != after.st_mtim.tv_nsec ||
before.st_ctim.tv_sec != after.st_ctim.tv_sec ||
before.st_ctim.tv_nsec != after.st_ctim.tv_nsec) {
errno = ESTALE;
goto out;
}
if (fcntl(sealed, F_ADD_SEALS, seals) < 0)
goto out;
char path[64];
snprintf(path, sizeof(path), "%sself/fd/%d",
proc_fd == AT_FDCWD ? "/proc/" : "", sealed);
readonly = openat(proc_fd, path, O_RDONLY | O_CLOEXEC);
if (readonly >= 0 && fcntl(readonly, F_GET_SEALS) != seals) {
close(readonly);
readonly = -1;
errno = EPERM;
}
out: {
int saved = errno;
close(sealed);
errno = saved;
return readonly;
}
}
static int sp_open_setup_script(int root_fd, const char *absolute_path,
int proc_fd) {
struct open_how how = {
.flags = O_RDONLY | O_NONBLOCK | O_CLOEXEC,
.resolve = RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS,
};
int source = (int)syscall(SYS_openat2, root_fd, absolute_path + 1,
&how, sizeof(how));
if (source < 0)
return -1;
int sealed = sp_copy_sealed_script(source, proc_fd);
int saved = errno;
close(source);
errno = saved;
return sealed;
}
static int sp_enter_final_root_empty(const struct fs_sandbox *sandbox);
static void prepare_fs_sandbox(struct fs_sandbox *sandbox,
const char *cwd, const char *base,
int allow_workspace_fallback,
const char *guest_script, int *script_fd) {
if (strcmp(cwd, "/") == 0) {
fprintf(stderr, "Refusing to share / as the sandbox workspace; run from a "
"project directory\n");
exit(1);
}
memcpy(sandbox->resolved_cwd, cwd, strlen(cwd) + 1);
sandbox->cwd_readonly_fallback = 0;
sandbox->readonly_binds_enforced = 1;
/* Create each staging directory once, relative to the staging tmpfs, instead
* of re-walking every prefix of base from "/". Parents precede children. */
static const struct {
const char *name;
mode_t mode;
} staging_dirs[] = {
{"proc", 0555}, {"rootfs", 0755},
{"sandbox", 0755}, {"dev-min", 0755},
};
if (realpath(base, sandbox->staging) == NULL)
die("resolve overlay base");
int base_fd = sp_open_directory_nosymlinks(sandbox->staging);
if (base_fd < 0)
die("open overlay base");
for (size_t i = 0; i < sizeof(staging_dirs) / sizeof(staging_dirs[0]); ++i) {
if (mkdirat(base_fd, staging_dirs[i].name, staging_dirs[i].mode) < 0 &&
errno != EEXIST)
die(staging_dirs[i].name);
}
close(base_fd);
{
char staging_proc[PATH_MAX];
sp_staging_path(sandbox, "/proc", staging_proc, sizeof(staging_proc));
if (mount("proc", staging_proc, "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC,
"hidepid=2") < 0)
die("mount staging proc");
}
setup_final_root(sandbox, allow_workspace_fallback);
int proc_fd = move_staging_proc(sandbox);
if (guest_script != NULL) {
*script_fd = sp_open_setup_script(sandbox->staging_root_fd, guest_script,
proc_fd);
if (*script_fd < 0)
die("acquire guest setup script");
}
close(proc_fd);
sp_close_fs_scopes(sandbox);
/* The one root switch: the final root becomes the namespace root, and the
* old root, which is the host tree with the staging tmpfs inside it, is
* detached as a whole. Nothing above the final root stays reachable, so a
* chroot escape from setup or the payload ends at the final root (C11 F1).
* This happens before the TUN handoff so the broker can remove the host
* scratch directory that the staging tmpfs was mounted on. */
if (sp_enter_final_root_empty(sandbox) == 0)
return;
if (chdir(sandbox->final_root) < 0 ||
syscall(SYS_pivot_root, ".", ".") < 0)
die("pivot final root");
if (umount2(".", MNT_DETACH) < 0)
die("detach host and staging trees");
if (chdir("/") < 0)
die("chdir final root");
}
#ifndef UNSHARE_EMPTY_MNTNS
#define UNSHARE_EMPTY_MNTNS 0x00100000 /* Linux 7.1 */
#endif
/* Root switch without pivot_root (Linux 7.1+). Clone the finished final tree
* into an anonymous mount tree, move into a new empty mount namespace (its
* root is an empty nullfs), attach the tree there and make it the root. Only
* the caller's root and cwd are reset, so there is no global task walk; the
* copied namespace holding the host tree and the staging tmpfs is released
* with PID 1's last reference to it. Returns -1 with nothing changed if the
* kernel or an outer policy rejects the empty namespace; the caller then
* uses pivot_root. */
static int sp_enter_final_root_empty(const struct fs_sandbox *sandbox) {
int tree_fd = (int)syscall(SYS_open_tree, AT_FDCWD, sandbox->final_root,
OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC |
AT_RECURSIVE);
int root_fd;
if (tree_fd < 0)
die("clone final root");
if (unshare(UNSHARE_EMPTY_MNTNS) < 0) {
int saved = errno;
close(tree_fd);
if (saved == EINVAL || saved == EPERM || saved == EACCES ||
saved == ENOSYS) {
DBG("empty mount namespace unavailable (%s); using pivot_root",
strerror(saved));
return -1;
}
errno = saved;
die("unshare empty mount namespace");
}
if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL) < 0)
die("make empty root private");
root_fd = open("/", O_PATH | O_DIRECTORY | O_CLOEXEC);
if (root_fd < 0 ||
syscall(SYS_move_mount, tree_fd, "", root_fd, "",
MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0 ||
fchdir(tree_fd) < 0 || chroot(".") < 0 || chdir("/") < 0)
die("enter final root");
close(root_fd);
close(tree_fd);
return 0;
}
static void enter_fs_sandbox(const struct fs_sandbox *sandbox) {
if (chdir(sandbox->resolved_cwd) < 0)
die("chdir sandbox cwd");
}
static int ensure_no_new_privs(void) {
return prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);
}
static uint64_t landlock_read_exec_rights(void) {
#if SP_HAVE_LANDLOCK
return LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_READ_FILE |
LANDLOCK_ACCESS_FS_READ_DIR;
#else
return 0;
#endif
}
static uint64_t landlock_write_rights_for_abi(int abi) {
uint64_t rights = 0;
#if SP_HAVE_LANDLOCK
(void)abi;
rights = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_REMOVE_DIR |
LANDLOCK_ACCESS_FS_REMOVE_FILE | LANDLOCK_ACCESS_FS_MAKE_CHAR |
LANDLOCK_ACCESS_FS_MAKE_DIR | LANDLOCK_ACCESS_FS_MAKE_REG |
LANDLOCK_ACCESS_FS_MAKE_SOCK | LANDLOCK_ACCESS_FS_MAKE_FIFO |
LANDLOCK_ACCESS_FS_MAKE_BLOCK | LANDLOCK_ACCESS_FS_MAKE_SYM;
#ifdef LANDLOCK_ACCESS_FS_REFER
if (abi >= 2)
rights |= LANDLOCK_ACCESS_FS_REFER;
#endif
#ifdef LANDLOCK_ACCESS_FS_TRUNCATE
if (abi >= 3)
rights |= LANDLOCK_ACCESS_FS_TRUNCATE;
#endif
#else
(void)abi;
#endif
return rights;
}
static int add_landlock_rule(int ruleset_fd, const char *path,
uint64_t allowed_access) {
#if SP_HAVE_LANDLOCK && defined(__NR_landlock_add_rule)
int dirfd = open(path, O_PATH | O_CLOEXEC);
if (dirfd < 0) {
int saved = errno;
fprintf(stderr, "[sockpuppet] Landlock open %s: %s\n", path,
strerror(saved));
errno = saved;
return -1;
}
struct landlock_path_beneath_attr rule = {
.allowed_access = allowed_access,
.parent_fd = dirfd,
};
int rc = (int)syscall(__NR_landlock_add_rule, ruleset_fd,
LANDLOCK_RULE_PATH_BENEATH, &rule, 0);
if (rc < 0) {
int saved = errno;
fprintf(stderr, "[sockpuppet] Landlock add rule %s: %s\n", path,
strerror(saved));
close(dirfd);
errno = saved;
return rc;
}
close(dirfd);
return rc;
#else
(void)ruleset_fd;
(void)path;
(void)allowed_access;
errno = ENOSYS;
return -1;
#endif
}
static int landlock_unavailable(const struct fs_sandbox *sandbox,
const char *reason) {
if (sandbox != NULL && !sandbox->readonly_binds_enforced) {
fprintf(stderr,
"[sockpuppet] Warning: read-only bind fallback requires Landlock "
"because one or more read-only remounts were unavailable (%s)\n",
reason);
errno = EOPNOTSUPP;
return -1;
}
(void)sp_stderr_printf("[sockpuppet] Warning: %s Landlock unavailable (%s); %s\n",
sandbox != NULL ? "child" : "parent", reason,
sandbox != NULL ? "mount-based filesystem confinement remains enforced"
: "broker filesystem-denial layer is absent");
return 0;
}
/* NULL selects the parent's deny-all policy; otherwise add the child's paths.
* Return 1 for installed, 0 for unavailable, -1 for a fatal policy error.
* Only the ABI query may classify kernel absence/disablement as unavailable;
* errors constructing or installing a supported policy are fatal. */
static int apply_landlock_policy(const struct fs_sandbox *sandbox) {
#if SP_HAVE_LANDLOCK && defined(__NR_landlock_create_ruleset) && \
defined(__NR_landlock_restrict_self)
int abi = (int)syscall(__NR_landlock_create_ruleset, NULL, 0,
LANDLOCK_CREATE_RULESET_VERSION);
if (abi < 0) {
if (errno == ENOSYS || errno == EOPNOTSUPP)
return landlock_unavailable(sandbox, strerror(errno));
return -1;
}
uint64_t read_exec = landlock_read_exec_rights();
uint64_t write_rights = landlock_write_rights_for_abi(abi);
struct landlock_ruleset_attr ruleset = {
.handled_access_fs = read_exec | write_rights,
};
int ruleset_fd = (int)syscall(__NR_landlock_create_ruleset, &ruleset,
sizeof(ruleset), 0);
if (ruleset_fd < 0)
return -1;
/* Read access below "/" covers /dev and its read-only devices; the rules
* below only add write access. */
if (sandbox != NULL &&
(add_landlock_rule(ruleset_fd, "/", read_exec) < 0 ||
add_landlock_rule(ruleset_fd, "/tmp", read_exec | write_rights) < 0 ||
add_landlock_rule(ruleset_fd, "/dev/pts",
LANDLOCK_ACCESS_FS_READ_DIR |
LANDLOCK_ACCESS_FS_READ_FILE |
LANDLOCK_ACCESS_FS_WRITE_FILE) < 0 ||
add_landlock_rule(ruleset_fd, "/dev/null",
LANDLOCK_ACCESS_FS_READ_FILE |
LANDLOCK_ACCESS_FS_WRITE_FILE) < 0 ||
/* /dev/tty only exists in interactive sandboxes (mount_minimal_dev). */
(interactive_stdio &&
add_landlock_rule(ruleset_fd, "/dev/tty",
LANDLOCK_ACCESS_FS_READ_FILE |
LANDLOCK_ACCESS_FS_WRITE_FILE) < 0) ||
add_landlock_rule(ruleset_fd, "/dev/ptmx",
LANDLOCK_ACCESS_FS_READ_FILE |
LANDLOCK_ACCESS_FS_WRITE_FILE) < 0)) {
close(ruleset_fd);
return -1;
}
if (sandbox != NULL) {
for (size_t i = 0; i < sandbox->scope_count; ++i) {
const struct sp_mount_scope *scope = &sandbox->scopes[i];
if (!scope->attached || scope->mode == SP_BIND_RO)
continue;
const char *path = scope->kind == SP_SCOPE_ROOTFS ? "/"
: scope->kind == SP_SCOPE_BIND ? scope->bind->target
: sandbox->resolved_cwd;
if (add_landlock_rule(ruleset_fd, path, read_exec | write_rights) < 0) {
close(ruleset_fd);
return -1;
}
}
}
if (sandbox == NULL) {
/* Verify the denial probe's target is readable before this policy. This
* avoids treating a missing file or an inherited restriction as evidence. */
int probe_fd = open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (probe_fd < 0) {
close(ruleset_fd);
return -1;
}
close(probe_fd);
}
if (ensure_no_new_privs() < 0 ||
syscall(__NR_landlock_restrict_self, ruleset_fd, 0) < 0) {
int saved = errno;
close(ruleset_fd);
errno = saved;
return -1;
}
close(ruleset_fd);
return 1;
#else
return landlock_unavailable(sandbox, "not compiled in");
#endif
}
/* Called only after this role's successful confinement/probe sequence. The
* parent's record does not attest to child setup or inherited capability sets. */
/* A caller's payload profile (--seccomp-file): raw struct sock_filter records
* for this architecture, read on the host before the sandbox clone, so PID 1
* gets it in its copy of memory and no descriptor crosses over. It replaces the
* default and binds PID 1 too. The floor goes first and keeps its prologue,
* which kills foreign-ABI and x32 calls whatever the profile returns. */
static struct sock_filter sp_profile[BPF_MAXINSNS];
static unsigned short sp_profile_len; /* 0: no profile, install the default */
static void startup_confinement_assurance(const struct fs_sandbox *sandbox,
int landlock) {
int child = sandbox != NULL;
#ifdef __NR_clone3
const char *clone3_probe = "passed";
#else
const char *clone3_probe = "unavailable-build";
#endif
(void)sp_stderr_printf(
"sp_startup_assurance\trole\t%s\tphase\tconfined\tuserns\t%s\t"
"mounts\t%s\treadonly\t%s\tlandlock\t%s\tlandlock_probe\t%s\t"
"seccomp\t%s\tseccomp_probe\t%s\tclone3_probe\t%s\t"
"caps_dropped\t%s\tno_new_privs\tset\tvnet_gso\tprofile-validated\t"
"tun_offload\tCSUM|TSO4\n",
child ? "child" : "parent", child ? "created" : "not-created",
child ? "enforced" : "not-applicable",
child ? (sandbox->readonly_binds_enforced ? "mounts" : "landlock-fallback")
: "not-applicable",
landlock > 0 ? "installed" : "unavailable",
!child && landlock > 0 ? "passed" : "not-probed",
child && sp_profile_len != 0 ? "profile" : "installed", child ? "not-probed" : "passed", child ? "not-probed" : clone3_probe,
child ? "cleared-verified" : "not-performed");
}
static int parent_harden_probe_landlock(void) {
errno = 0;
int fd = open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (fd >= 0) {
close(fd);
fprintf(stderr,
"sockpuppet: parent hardening probe failed: Landlock allowed "
"open(/)\n");
errno = EPERM;
return -1;
}
if (errno != EACCES && errno != EPERM) {
int saved = errno;
fprintf(stderr,
"sockpuppet: parent hardening probe failed: open(/) "
"returned %s\n",
strerror(saved));
errno = saved;
return -1;
}
(void)sp_stderr_printf(
"sockpuppet: parent hardening probe: Landlock denied open\n");
return 0;
}
static int parent_harden_expect_denied(const char *name, long result) {
int saved = errno;
if (result == -1 && saved == EPERM)
return 0;
(void)sp_stderr_printf(
"sockpuppet: parent hardening probe failed: %s denial was %s (rc=%ld)\n",
name, strerror(saved), result);
errno = saved ? saved : EPERM;
return -1;
}
/* Always-on post-seccomp probes. The extended argument-level matrix lives in
* tests/tcp_refactor/seccomp_policy.c, which installs the same built filter
* through apply_parent_seccomp() in a forked process. */
static int parent_harden_probe_seccomp(void) {
/* Invalid arguments make these safe even if the filter is broken: no child
* can be created and no socket can be changed. Filter denial must be EPERM,
* not the syscall's own EBADF/EINVAL/ENOSYS validation result. */
#ifdef __NR_clone3
errno = 0;
if (parent_harden_expect_denied("clone3",
syscall(__NR_clone3, NULL, 0)) < 0)
return -1;
(void)sp_stderr_printf(
"sockpuppet: parent hardening probe: seccomp denied clone3\n");
#else
(void)sp_stderr_printf(
"sockpuppet: parent hardening probe: clone3 unavailable at build time\n");
#endif
int one = 1;
errno = 0;
if (parent_harden_expect_denied(
"SO_KEEPALIVE", setsockopt(-1, SOL_SOCKET, SO_KEEPALIVE,
&one, sizeof(one))) < 0)
return -1;
(void)sp_stderr_printf(
"sockpuppet: parent hardening probe: seccomp denied SO_KEEPALIVE\n");
return 0;
}
#ifndef SECCOMP_RET_KILL_PROCESS
#define SECCOMP_RET_KILL_PROCESS SECCOMP_RET_KILL
#endif
#ifndef __X32_SYSCALL_BIT
#define __X32_SYSCALL_BIT 0x40000000U
#endif
#define SP_CLONE_NAMESPACE_FLAGS_BASE \
(CLONE_NEWUSER | CLONE_NEWNET | CLONE_NEWNS | CLONE_NEWIPC | \
CLONE_NEWUTS | CLONE_NEWPID)
#ifdef CLONE_NEWCGROUP
#define SP_CLONE_NAMESPACE_FLAGS_CGROUP | CLONE_NEWCGROUP
#else
#define SP_CLONE_NAMESPACE_FLAGS_CGROUP
#endif
#ifdef CLONE_NEWTIME
#define SP_CLONE_NAMESPACE_FLAGS_TIME | CLONE_NEWTIME
#else
#define SP_CLONE_NAMESPACE_FLAGS_TIME
#endif
#define SP_CLONE_NAMESPACE_FLAGS \
(SP_CLONE_NAMESPACE_FLAGS_BASE SP_CLONE_NAMESPACE_FLAGS_CGROUP \
SP_CLONE_NAMESPACE_FLAGS_TIME)
#ifdef CLONE_UNTRACED
#define SP_CLONE_UNTRACED_FLAG CLONE_UNTRACED
#else
#define SP_CLONE_UNTRACED_FLAG 0
#endif
#define SP_CLONE_BLOCKED_FLAGS \
(SP_CLONE_NAMESPACE_FLAGS | SP_CLONE_UNTRACED_FLAG)
#define SP_SECCOMP_KILL SECCOMP_RET_KILL_PROCESS
#ifndef AUDIT_ARCH_AARCH64
#define AUDIT_ARCH_AARCH64 \
(__AUDIT_ARCH_64BIT | __AUDIT_ARCH_LE | 183)
#endif
#ifndef AUDIT_ARCH_RISCV64
#define AUDIT_ARCH_RISCV64 \
(__AUDIT_ARCH_64BIT | __AUDIT_ARCH_LE | 243)
#endif
#if defined(__x86_64__)
#define SP_AUDIT_ARCH AUDIT_ARCH_X86_64
#elif defined(__aarch64__)
#define SP_AUDIT_ARCH AUDIT_ARCH_AARCH64
#elif defined(__riscv) && __riscv_xlen == 64
#define SP_AUDIT_ARCH AUDIT_ARCH_RISCV64
#else
#define SP_AUDIT_ARCH 0
#endif
/* Instructions for built-in filter templates. The prologue kills a call from a
* foreign ABI (on x86_64 also an x32 number) and leaves the syscall number in
* the accumulator; each rule returns `action` for syscall `nr` and otherwise
* falls through to the next rule, so rules never jump across one another. */
#define SP_BPF_LOAD(field) BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, field))
#define SP_BPF_RET(action) BPF_STMT(BPF_RET | BPF_K, (action))
#if defined(__x86_64__)
#define SP_BPF_ARCH_PROLOGUE \
SP_BPF_LOAD(arch), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SP_AUDIT_ARCH, 1, 0), \
SP_BPF_RET(SP_SECCOMP_KILL), SP_BPF_LOAD(nr), \
BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, __X32_SYSCALL_BIT, 0, 1), SP_BPF_RET(SP_SECCOMP_KILL)
#else
#define SP_BPF_ARCH_PROLOGUE \
SP_BPF_LOAD(arch), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SP_AUDIT_ARCH, 1, 0), \
SP_BPF_RET(SP_SECCOMP_KILL), SP_BPF_LOAD(nr)
#endif
#define SP_BPF_NR_RET(nr, action) \
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (nr), 0, 1), SP_BPF_RET(action)
/* Loads the high 32 bits of a 64-bit argument (all supported ABIs are LE). */
#define SP_BPF_LOAD_HIGH(field) \
BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, field) + sizeof(uint32_t))
/* Returns `action` when `syscall`'s argument `arg` has low 32 bits equal to
* `value`; otherwise reloads the syscall number and falls through. */
#define SP_BPF_ARG_EQ_RET(syscall, arg, value, action) \
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (syscall), 0, 3), SP_BPF_LOAD(args[arg]), \
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (value), 0, 1), SP_BPF_RET(action), SP_BPF_LOAD(nr)
/* The kernel copies a program at install; it never writes through `filter`. */
#define SP_BPF_PROGRAM(template) \
((struct sock_fprog){.len = sizeof(template) / sizeof((template)[0]), \
.filter = (struct sock_filter *)(template)})
enum {
SP_PARENT_SECCOMP_MAX_FILTER = 256,
SP_PARENT_SECCOMP_ERRNO = SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA),
/* Linux termios2 ioctl values. glibc may use these instead of the legacy
* TCSETS family when the saved termios carries explicit input/output
* speeds. They have the same UAPI values on the supported 64-bit Linux
* architectures (x86_64, aarch64, and riscv64). */
SP_IOCTL_TCSETS2 = 0x402c542bU,
SP_IOCTL_TCSETSW2 = 0x402c542cU,
SP_IOCTL_TCSETSF2 = 0x402c542dU,
};
#define SP_BPF_JEQ(value, jt, jf) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (value), (jt), (jf))
#define SP_PARENT_DENY SP_BPF_RET(SP_PARENT_SECCOMP_ERRNO)
#define SP_PARENT_ALLOW(nr) SP_BPF_NR_RET(nr, SECCOMP_RET_ALLOW)
/* Allows the call when argument a1 equals first and a2 equals second (low
* words); otherwise falls through to the next pair. */
#define SP_BPF_ALLOW_PAIR(a1, first, a2, second) \
SP_BPF_LOAD(args[a1]), SP_BPF_JEQ(first, 0, 3), SP_BPF_LOAD(args[a2]), \
SP_BPF_JEQ(second, 0, 1), SP_BPF_RET(SECCOMP_RET_ALLOW)
#if SP_AUDIT_ARCH != 0
/* Broker policy: allow a named set, deny the rest with EPERM. The filter is
* assembled at install from fixed segments, in this order; a disabled option
* contributes no segment, and runtime values are written into named slots.
* Each segment starts with "not this syscall: skip the segment", and every
* other jump stays inside its segment. */
static const struct sock_filter sp_parent_head[] = {
SP_BPF_ARCH_PROLOGUE,
SP_PARENT_ALLOW(__NR_read), SP_PARENT_ALLOW(__NR_write),
SP_PARENT_ALLOW(__NR_readv), SP_PARENT_ALLOW(__NR_writev),
SP_PARENT_ALLOW(__NR_close), SP_PARENT_ALLOW(__NR_exit),
SP_PARENT_ALLOW(__NR_exit_group), SP_PARENT_ALLOW(__NR_wait4),
SP_PARENT_ALLOW(__NR_epoll_ctl), SP_PARENT_ALLOW(__NR_epoll_pwait),
SP_PARENT_ALLOW(__NR_fcntl), SP_PARENT_ALLOW(__NR_getrandom),
SP_PARENT_ALLOW(__NR_clock_gettime), SP_PARENT_ALLOW(__NR_rt_sigreturn),
SP_PARENT_ALLOW(__NR_restart_syscall), SP_PARENT_ALLOW(__NR_accept4),
SP_PARENT_ALLOW(__NR_getpeername), SP_PARENT_ALLOW(__NR_recvfrom),
SP_PARENT_ALLOW(__NR_recvmsg), SP_PARENT_ALLOW(__NR_sendto),
SP_PARENT_ALLOW(__NR_sendmsg), SP_PARENT_ALLOW(__NR_shutdown),
/* x86_64 has no recv/send; the generic ABI (aarch64, riscv64) has no
* epoll_wait or time; epoll_pwait2 needs 5.11 headers. */
#ifdef __NR_epoll_wait
SP_PARENT_ALLOW(__NR_epoll_wait),
#endif
#ifdef __NR_epoll_pwait2
SP_PARENT_ALLOW(__NR_epoll_pwait2),
#endif
#ifdef __NR_time
SP_PARENT_ALLOW(__NR_time),
#endif
#ifdef __NR_recv
SP_PARENT_ALLOW(__NR_recv),
#endif
#ifdef __NR_send
SP_PARENT_ALLOW(__NR_send),
#endif
};
/* Shutdown may release exactly the startup map arena; mmap/mremap and any
* other unmap stay denied. Its address is written into the two slots. */
enum { SP_PARENT_MUNMAP_LOW = 2, SP_PARENT_MUNMAP_HIGH = 5 };
static const struct sock_filter sp_parent_munmap[] = {
SP_BPF_JEQ(__NR_munmap, 0, 13),
SP_BPF_LOAD(args[0]), SP_BPF_JEQ(0, 1, 0), SP_PARENT_DENY,
SP_BPF_LOAD_HIGH(args[0]), SP_BPF_JEQ(0, 1, 0), SP_PARENT_DENY,
SP_BPF_LOAD(args[1]), SP_BPF_JEQ(sizeof(struct sp_map_storage), 1, 0), SP_PARENT_DENY,
SP_BPF_LOAD_HIGH(args[1]), SP_BPF_JEQ(0, 1, 0), SP_PARENT_DENY,
SP_BPF_RET(SECCOMP_RET_ALLOW),
};
/* Fatal-teardown signalling: SIGKILL to exactly the namespace child, by
* pidfd_send_signal() or kill(). The syscall and target go into the slots. */
enum { SP_PARENT_SIGNAL_NR = 0, SP_PARENT_SIGNAL_TARGET = 2 };
static const struct sock_filter sp_parent_signal[] = {
SP_BPF_JEQ(0, 0, 6),
SP_BPF_LOAD(args[0]), SP_BPF_JEQ(0, 0, 2),
SP_BPF_LOAD(args[1]), SP_BPF_JEQ(SIGKILL, 1, 0),
SP_PARENT_DENY, SP_BPF_RET(SECCOMP_RET_ALLOW),
};
/* Interactive sessions: window size and terminal mode ioctls only. The
* TCSETS*2 values are glibc's termios2 requests (same on all supported ABIs). */
static const struct sock_filter sp_parent_ioctl[] = {
SP_BPF_JEQ(__NR_ioctl, 0, 11),
SP_BPF_LOAD(args[1]),
SP_BPF_JEQ(TIOCGWINSZ, 8, 0), SP_BPF_JEQ(TIOCSWINSZ, 7, 0),
SP_BPF_JEQ(TCSETS, 6, 0), SP_BPF_JEQ(TCSETSW, 5, 0),
SP_BPF_JEQ(TCSETSF, 4, 0), SP_BPF_JEQ(SP_IOCTL_TCSETS2, 3, 0),
SP_BPF_JEQ(SP_IOCTL_TCSETSW2, 2, 0), SP_BPF_JEQ(SP_IOCTL_TCSETSF2, 1, 0),
SP_PARENT_DENY, SP_BPF_RET(SECCOMP_RET_ALLOW),
};
/* Host sockets: the option, mmsg and readback calls the broker makes itself. */
enum { SP_SETSOCKOPT_PAIRS = 5, SP_GETSOCKOPT_PAIRS = 10 };
static const struct sock_filter sp_parent_sockets[] = {
/* setsockopt: int-sized values only, for these (level, option) pairs. */
SP_BPF_JEQ(__NR_setsockopt, 0, 3 + 5 * SP_SETSOCKOPT_PAIRS),
SP_BPF_LOAD(args[4]), SP_BPF_JEQ(sizeof(int), 0, 5 * SP_SETSOCKOPT_PAIRS),
SP_BPF_ALLOW_PAIR(1, IPPROTO_TCP, 2, TCP_NODELAY),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_SNDBUF),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_RCVBUF),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_RCVLOWAT),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_PEEK_OFF),
SP_PARENT_DENY,
SP_BPF_JEQ(__NR_getsockopt, 0, 1 + 5 * SP_GETSOCKOPT_PAIRS),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_ERROR),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_DOMAIN),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_TYPE),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_PROTOCOL),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_SNDBUF),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_RCVBUF),
SP_BPF_ALLOW_PAIR(1, IPPROTO_TCP, 2, TCP_NODELAY),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_RCVLOWAT),
SP_BPF_ALLOW_PAIR(1, SOL_SOCKET, 2, SO_PEEK_OFF),
SP_BPF_ALLOW_PAIR(1, IPPROTO_IP, 2, IP_MTU),
SP_PARENT_DENY,
/* recvmmsg: at most SP_UDP_MMSG_MAX messages, these flags, no timeout. */
SP_BPF_JEQ(__NR_recvmmsg, 0, 10),
SP_BPF_LOAD(args[2]), BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, SP_UDP_MMSG_MAX, 6, 0),
SP_BPF_LOAD(args[3]), SP_BPF_JEQ(MSG_DONTWAIT | MSG_CMSG_CLOEXEC, 0, 4),
SP_BPF_LOAD(args[4]), SP_BPF_JEQ(0, 0, 2),
SP_BPF_LOAD_HIGH(args[4]), SP_BPF_JEQ(0, 1, 0),
SP_PARENT_DENY, SP_BPF_RET(SECCOMP_RET_ALLOW),
/* sendmmsg: at most SP_UDP_MMSG_MAX messages, these flags. */
SP_BPF_JEQ(__NR_sendmmsg, 0, 6),
SP_BPF_LOAD(args[2]), BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, SP_UDP_MMSG_MAX, 2, 0),
SP_BPF_LOAD(args[3]), SP_BPF_JEQ(MSG_DONTWAIT | MSG_NOSIGNAL, 1, 0),
SP_PARENT_DENY, SP_BPF_RET(SECCOMP_RET_ALLOW),
};
_Static_assert(sizeof(sp_parent_sockets) / sizeof(sp_parent_sockets[0]) ==
4 + 5 * SP_SETSOCKOPT_PAIRS + 2 + 5 * SP_GETSOCKOPT_PAIRS + 11 + 7,
"socket option pair counts must match the segment");
/* Direct egress: IPv4 stream/datagram sockets (CLOEXEC/NONBLOCK allowed),
* then connect, bind and getsockname. */
static const struct sock_filter sp_parent_outbound[] = {
SP_BPF_JEQ(__NR_socket, 0, 8),
SP_BPF_LOAD(args[0]), SP_BPF_JEQ(AF_INET, 0, 4),
SP_BPF_LOAD(args[1]),
BPF_STMT(BPF_ALU | BPF_AND | BPF_K, ~((unsigned int)SOCK_CLOEXEC | (unsigned int)SOCK_NONBLOCK)),
SP_BPF_JEQ(SOCK_STREAM, 2, 0), SP_BPF_JEQ(SOCK_DGRAM, 1, 0),
SP_PARENT_DENY, SP_BPF_RET(SECCOMP_RET_ALLOW),
SP_PARENT_ALLOW(__NR_connect), SP_PARENT_ALLOW(__NR_bind), SP_PARENT_ALLOW(__NR_getsockname),
};
static size_t sp_bpf_append(struct sock_filter *out, size_t used, const struct sock_filter *segment,
size_t count) {
assert(count <= SP_PARENT_SECCOMP_MAX_FILTER - used);
memcpy(out + used, segment, count * sizeof(*segment));
return used + count;
}
/* Writes a runtime value into a comparison slot of a just-copied segment. */
static void sp_bpf_set_slot(struct sock_filter *insn, uint32_t value) {
assert(insn->code == (BPF_JMP | BPF_JEQ | BPF_K));
insn->k = value;
}
#define SP_BPF_APPEND(out, used, segment) \
sp_bpf_append(out, used, segment, sizeof(segment) / sizeof((segment)[0]))
static size_t sp_parent_append_signal(struct sock_filter *out, size_t used, unsigned int nr,
unsigned int target) {
size_t next = SP_BPF_APPEND(out, used, sp_parent_signal);
sp_bpf_set_slot(&out[used + SP_PARENT_SIGNAL_NR], nr);
sp_bpf_set_slot(&out[used + SP_PARENT_SIGNAL_TARGET], target);
return next;
}
/* Assembles the broker filter for cfg into out[SP_PARENT_SECCOMP_MAX_FILTER]
* and returns its length. Targets compare low words, as the kernel's
* seccomp_data does for int arguments. */
static unsigned short sp_parent_seccomp_filter(const struct parent_harden_config *cfg,
struct sock_filter *out) {
size_t used = SP_BPF_APPEND(out, 0, sp_parent_head);
if (g_map_storage != NULL) {
uint64_t address = (uint64_t)(uintptr_t)g_map_storage;
size_t at = used;
used = SP_BPF_APPEND(out, used, sp_parent_munmap);
sp_bpf_set_slot(&out[at + SP_PARENT_MUNMAP_LOW], (uint32_t)address);
sp_bpf_set_slot(&out[at + SP_PARENT_MUNMAP_HIGH], (uint32_t)(address >> 32));
}
#ifdef __NR_pidfd_send_signal
if (cfg->child_pidfd >= 0)
used = sp_parent_append_signal(out, used, __NR_pidfd_send_signal,
(unsigned int)cfg->child_pidfd);
#endif
if (cfg->child_pid > 0)
used = sp_parent_append_signal(out, used, __NR_kill, (unsigned int)cfg->child_pid);
if (cfg->interactive)
used = SP_BPF_APPEND(out, used, sp_parent_ioctl);
used = SP_BPF_APPEND(out, used, sp_parent_sockets);
if (cfg->outbound_sockets_allowed)
used = SP_BPF_APPEND(out, used, sp_parent_outbound);
out[used++] = (struct sock_filter)SP_PARENT_DENY;
return (unsigned short)used;
}
#endif
#ifndef SECCOMP_FILTER_FLAG_LOG
#define SECCOMP_FILTER_FLAG_LOG (1UL << 1)
#endif
/* Filters go through seccomp(2), not prctl: only it takes FLAG_LOG, which puts
* ERRNO denials in the kernel log beside kills (type=1326, with the syscall
* number), so a syscall a newer libc starts using is easy to find. */
static int sp_seccomp_install(const struct sock_fprog *program, unsigned long flags) {
return (int)syscall(__NR_seccomp, SECCOMP_SET_MODE_FILTER, flags, program);
}
static int apply_parent_seccomp(const struct parent_harden_config *cfg) {
#if SP_AUDIT_ARCH != 0
struct sock_filter filter[SP_PARENT_SECCOMP_MAX_FILTER];
struct sock_fprog program = {.len = sp_parent_seccomp_filter(cfg, filter), .filter = filter};
if (ensure_no_new_privs() < 0)
return -1;
/* Not logged: parent_harden_probe_seccomp() is denied on every launch. */
return sp_seccomp_install(&program, 0);
#else
(void)cfg;
fprintf(stderr, "sockpuppet: parent seccomp unsupported on this architecture\n");
errno = ENOSYS;
return -1;
#endif
}
/* Payload policy: deny a named set, allow everything else, as two installed
* filters. The floor holds the rules no payload profile (plan S2) may relax;
* the default holds the rest, which a profile would replace. The kernel runs
* both and keeps the stricter action; their rules name disjoint syscalls, so
* the pair decides exactly as one combined filter would. */
#define SP_CHILD_KILL SP_SECCOMP_KILL
#define SP_CHILD_EPERM (SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA))
#if SP_AUDIT_ARCH != 0
static const struct sock_filter sp_payload_floor_filter[] = {
SP_BPF_ARCH_PROLOGUE,
/* Kernel code and tracing, keyrings, cross-process memory and mounts. */
SP_BPF_NR_RET(__NR_init_module, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_finit_module, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_delete_module, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_kexec_load, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_kexec_file_load, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_bpf, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_perf_event_open, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_userfaultfd, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_keyctl, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_add_key, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_request_key, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_process_vm_readv, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_process_vm_writev, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_mount, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_umount2, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_pivot_root, SP_CHILD_KILL),
/* fspick, open_tree_attr and io_uring may be absent from older headers;
* the rest of the new mount API is required by the mount code. */
SP_BPF_NR_RET(__NR_open_tree, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_move_mount, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_fsopen, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_fsconfig, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_fsmount, SP_CHILD_KILL),
#ifdef __NR_fspick
SP_BPF_NR_RET(__NR_fspick, SP_CHILD_KILL),
#endif
SP_BPF_NR_RET(__NR_mount_setattr, SP_CHILD_KILL),
#ifdef __NR_open_tree_attr
SP_BPF_NR_RET(__NR_open_tree_attr, SP_CHILD_KILL),
#endif
#ifdef __NR_io_uring_setup
SP_BPF_NR_RET(__NR_io_uring_setup, SP_CHILD_EPERM),
#endif
#ifdef __NR_io_uring_enter
SP_BPF_NR_RET(__NR_io_uring_enter, SP_CHILD_EPERM),
#endif
#ifdef __NR_io_uring_register
SP_BPF_NR_RET(__NR_io_uring_register, SP_CHILD_EPERM),
#endif
SP_BPF_RET(SECCOMP_RET_ALLOW),
};
/* Namespaces, tracing, filter installation and tty injection. Installed after
* the floor: it denies prctl(PR_SET_SECCOMP), so nothing can follow it. */
static const struct sock_filter sp_payload_default_filter[] = {
SP_BPF_ARCH_PROLOGUE,
SP_BPF_NR_RET(__NR_unshare, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_setns, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_ptrace, SP_CHILD_KILL),
SP_BPF_NR_RET(__NR_seccomp, SP_CHILD_EPERM),
#ifdef __NR_clone3
/* ENOSYS, not EPERM: glibc falls back to clone(2) on ENOSYS. */
SP_BPF_NR_RET(__NR_clone3, SECCOMP_RET_ERRNO | (ENOSYS & SECCOMP_RET_DATA)),
#endif
SP_BPF_ARG_EQ_RET(__NR_prctl, 0, PR_SET_SECCOMP, SP_CHILD_EPERM),
SP_BPF_ARG_EQ_RET(__NR_socket, 0, AF_ALG, SP_CHILD_EPERM),
/* Deny the two tty-push ioctls, allow other ioctls; one argument load. */
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_ioctl, 0, 6), SP_BPF_LOAD(args[1]),
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, TIOCSTI, 0, 1), SP_BPF_RET(SP_CHILD_EPERM),
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, TIOCLINUX, 0, 1), SP_BPF_RET(SP_CHILD_EPERM),
SP_BPF_RET(SECCOMP_RET_ALLOW), SP_BPF_LOAD(nr),
/* Legacy clone(2): a nonzero high flags word, or any forbidden flag in the
* low word, is fatal. The high word is checked first so a 64-bit flag set
* cannot smuggle a blocked bit past a 32-bit mask. */
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_clone, 0, 7), SP_BPF_LOAD_HIGH(args[0]),
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), SP_BPF_RET(SP_SECCOMP_KILL),
SP_BPF_LOAD(args[0]), BPF_STMT(BPF_ALU | BPF_AND | BPF_K, SP_CLONE_BLOCKED_FLAGS),
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), SP_BPF_RET(SP_SECCOMP_KILL),
SP_BPF_RET(SECCOMP_RET_ALLOW),
};
_Static_assert(sizeof(sp_payload_default_filter) / sizeof(sp_payload_default_filter[0]) <=
BPF_MAXINSNS,
"payload default seccomp template exceeds BPF_MAXINSNS");
#endif
/* Parses n records in sp_profile. A profile must open with this
* architecture's check (one built for another ABI numbers syscalls
* differently) and return only constant actions sockpuppet can honour: it has
* no tracer, listener or SIGSYS handler for TRACE, USER_NOTIF or TRAP. The
* kernel checks the rest at install. Returns NULL or the reason. */
static const char *sp_profile_parse(size_t n) {
const struct sock_filter *f = sp_profile;
if (n < 2 || f[0].code != (BPF_LD | BPF_W | BPF_ABS) ||
f[0].k != offsetof(struct seccomp_data, arch) || f[1].code != (BPF_JMP | BPF_JEQ | BPF_K) ||
f[1].k != SP_AUDIT_ARCH)
return "does not start with this architecture's check (ld arch; jeq AUDIT_ARCH)";
for (size_t i = 0; i < n; ++i) {
uint32_t action = f[i].k & SECCOMP_RET_ACTION_FULL;
if (BPF_CLASS(f[i].code) != BPF_RET)
continue;
if (BPF_RVAL(f[i].code) != BPF_K)
return "returns a computed action";
if (action != SECCOMP_RET_ALLOW && action != SECCOMP_RET_ERRNO && action != SECCOMP_RET_LOG &&
action != SECCOMP_RET_KILL_PROCESS && action != SECCOMP_RET_KILL_THREAD)
return "returns an action other than ALLOW, ERRNO, LOG or KILL";
}
sp_profile_len = (unsigned short)n;
return NULL;
}
/* Reads --seccomp-file into sp_profile. Returns NULL or the reason. */
static const char *sp_profile_load(const char *path) {
const char *error;
struct stat st;
ssize_t got;
int fd;
if (SP_AUDIT_ARCH == 0)
return "seccomp is unsupported on this architecture";
if (path[0] != '/')
return "must be an absolute path";
fd = open(path, O_RDONLY | O_CLOEXEC | O_NOCTTY | O_NONBLOCK);
if (fd < 0)
return strerror(errno);
if (fstat(fd, &st) < 0)
error = strerror(errno);
else if (!S_ISREG(st.st_mode))
error = "not a regular file";
else if (st.st_size <= 0 || (uint64_t)st.st_size % sizeof(struct sock_filter) != 0)
error = "size is not a whole number of struct sock_filter records";
else if ((uint64_t)st.st_size > sizeof(sp_profile))
error = "longer than BPF_MAXINSNS instructions";
else if ((got = read(fd, sp_profile, (size_t)st.st_size)) != st.st_size)
error = got < 0 ? strerror(errno) : "short read";
else
error = sp_profile_parse((size_t)st.st_size / sizeof(struct sock_filter));
close(fd);
return error;
}
static int apply_child_seccomp(void) {
#if SP_AUDIT_ARCH != 0
struct sock_fprog floor = SP_BPF_PROGRAM(sp_payload_floor_filter);
struct sock_fprog def = SP_BPF_PROGRAM(sp_payload_default_filter);
struct sock_fprog profile = {.len = sp_profile_len, .filter = sp_profile};
if (ensure_no_new_privs() < 0 || sp_seccomp_install(&floor, SECCOMP_FILTER_FLAG_LOG) < 0)
return -1;
if (sp_profile_len != 0)
return sp_seccomp_install(&profile, SECCOMP_FILTER_FLAG_LOG);
/* Not logged: glibc tries clone3 (ENOSYS here) once in every process that
* starts a thread or calls posix_spawn, so a build would log per process. */
return sp_seccomp_install(&def, 0);
#else
fprintf(stderr, "[sockpuppet] Warning: Seccomp not supported on this "
"architecture, continuing without syscall filtering\n");
return 0;
#endif
}
/* Setup is a namespace-root program selected by the user. Keep its policy
* broad enough for package managers and ordinary services; deny only the
* operations that would change the fixed sandbox topology or add a second
* kernel-control path. The final payload policy remains separate. */
#define SP_SETUP_SECCOMP_ERRNO \
(SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA))
#if SP_AUDIT_ARCH != 0
static const struct sock_filter sp_setup_seccomp_filter[] = {
SP_BPF_ARCH_PROLOGUE,
/* fspick and open_tree_attr may be absent from older headers; the rest of
* the new mount API is required by the mount code. */
SP_BPF_NR_RET(__NR_mount, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_umount2, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_pivot_root, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_unshare, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_setns, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_ptrace, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_bpf, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_open_tree, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_move_mount, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_fsopen, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_fsconfig, SP_SETUP_SECCOMP_ERRNO),
SP_BPF_NR_RET(__NR_fsmount, SP_SETUP_SECCOMP_ERRNO),
#ifdef __NR_fspick
SP_BPF_NR_RET(__NR_fspick, SP_SETUP_SECCOMP_ERRNO),
#endif
SP_BPF_NR_RET(__NR_mount_setattr, SP_SETUP_SECCOMP_ERRNO),
#ifdef __NR_open_tree_attr
SP_BPF_NR_RET(__NR_open_tree_attr, SP_SETUP_SECCOMP_ERRNO),
#endif
SP_BPF_RET(SECCOMP_RET_ALLOW),
};
_Static_assert(sizeof(sp_setup_seccomp_filter) / sizeof(sp_setup_seccomp_filter[0]) <=
BPF_MAXINSNS,
"setup seccomp template exceeds BPF_MAXINSNS");
#endif
static int apply_setup_seccomp(void) {
#if SP_AUDIT_ARCH != 0
struct sock_fprog program = SP_BPF_PROGRAM(sp_setup_seccomp_filter);
if (ensure_no_new_privs() < 0)
return -1;
return sp_seccomp_install(&program, SECCOMP_FILTER_FLAG_LOG);
#else
errno = ENOSYS;
return -1;
#endif
}
static int sp_wait_status_to_exit_code(int status) {
if (WIFEXITED(status))
return WEXITSTATUS(status);
if (WIFSIGNALED(status))
return 128 + WTERMSIG(status);
return 1;
}
enum sp_pid1_phase {
SP_PID1_PHASE_NONE,
SP_PID1_PHASE_PREPARED,
SP_PID1_PHASE_BROKER_WAIT,
SP_PID1_PHASE_SETUP_RUNNING,
SP_PID1_PHASE_SETUP_REAPED,
SP_PID1_PHASE_FINAL_IDENTITY,
SP_PID1_PHASE_PAYLOAD_RUNNING,
SP_PID1_PHASE_TEARDOWN,
};
static int sp_pid1_phase_enter(enum sp_pid1_phase *current,
enum sp_pid1_phase next) {
static const char *const names[] = {
[SP_PID1_PHASE_PREPARED] = "prepared",
[SP_PID1_PHASE_BROKER_WAIT] = "broker_wait",
[SP_PID1_PHASE_SETUP_RUNNING] = "setup_running",
[SP_PID1_PHASE_SETUP_REAPED] = "setup_reaped",
[SP_PID1_PHASE_FINAL_IDENTITY] = "final_identity",
[SP_PID1_PHASE_PAYLOAD_RUNNING] = "payload_running",
[SP_PID1_PHASE_TEARDOWN] = "teardown",
};
int allowed = 0;
if (current == NULL || *current < SP_PID1_PHASE_NONE ||
*current > SP_PID1_PHASE_TEARDOWN || next <= SP_PID1_PHASE_NONE ||
next > SP_PID1_PHASE_TEARDOWN) {
errno = EPROTO;
return -1;
}
switch (*current) {
case SP_PID1_PHASE_NONE:
allowed = next == SP_PID1_PHASE_PREPARED;
break;
case SP_PID1_PHASE_PREPARED:
allowed = next == SP_PID1_PHASE_BROKER_WAIT ||
next == SP_PID1_PHASE_FINAL_IDENTITY;
break;
case SP_PID1_PHASE_BROKER_WAIT:
allowed = next == SP_PID1_PHASE_SETUP_RUNNING;
break;
case SP_PID1_PHASE_SETUP_RUNNING:
allowed = next == SP_PID1_PHASE_SETUP_REAPED;
break;
case SP_PID1_PHASE_SETUP_REAPED:
allowed = next == SP_PID1_PHASE_FINAL_IDENTITY;
break;
case SP_PID1_PHASE_FINAL_IDENTITY:
allowed = next == SP_PID1_PHASE_PAYLOAD_RUNNING;
break;
case SP_PID1_PHASE_PAYLOAD_RUNNING:
allowed = next == SP_PID1_PHASE_TEARDOWN;
break;
case SP_PID1_PHASE_TEARDOWN:
break;
}
if (!allowed) {
errno = EPROTO;
return -1;
}
*current = next;
bench_mark("pid1_phase", names[next], "entered", NULL);
return 0;
}
static int command_is_shell_without_args(int argc, char *const argv[]) {
static const char *const shell_names[] = {
"sh", "bash", "dash", "ash", "zsh", "ksh", "mksh", "fish",
};
const char *name;
const char *slash;
if (argc != 1 || argv == NULL || argv[0] == NULL || argv[0][0] == '\0')
return 0;
slash = strrchr(argv[0], '/');
name = slash != NULL ? slash + 1 : argv[0];
for (size_t i = 0; i < sizeof(shell_names) / sizeof(shell_names[0]); ++i) {
if (strcmp(name, shell_names[i]) == 0)
return 1;
}
return 0;
}
static void maybe_print_noninteractive_shell_hint(int argc, char *const argv[],
int status) {
if (interactive_stdio || !WIFEXITED(status) || WEXITSTATUS(status) != 0 ||
!command_is_shell_without_args(argc, argv))
return;
fprintf(stderr,
"[sockpuppet] Hint: %s exited because noninteractive mode gives the "
"payload /dev/null as stdin. Re-run with --interactive for a shell.\n",
argv[0]);
}
static void sp_setup_exec_child(int script_fd, int stdin_fd, int stdout_fd,
int stderr_fd, pid_t expected_parent) {
static char path_env[] = "PATH=/usr/sbin:/usr/bin:/sbin:/bin";
static char home_env[] = "HOME=/";
static char locale_env[] = "LC_ALL=C";
char script_path[64];
char *const envp[] = {path_env, home_env, locale_env, NULL};
char *const argv[] = {"/bin/sh", script_path, NULL};
arm_parent_death_signal(expected_parent);
if (dup2(stdin_fd, STDIN_FILENO) < 0 ||
dup2(stdout_fd, STDOUT_FILENO) < 0 ||
dup2(stderr_fd, STDERR_FILENO) < 0) {
dprintf(STDERR_FILENO, "[sockpuppet] setup stdio: %s\n", strerror(errno));
_exit(126);
}
if (stdin_fd > STDERR_FILENO)
close(stdin_fd);
if (stdout_fd > STDERR_FILENO && stdout_fd != stdin_fd)
close(stdout_fd);
if (stderr_fd > STDERR_FILENO && stderr_fd != stdin_fd &&
stderr_fd != stdout_fd)
close(stderr_fd);
if (script_fd != 3) {
if (dup3(script_fd, 3, 0) < 0) {
dprintf(STDERR_FILENO, "[sockpuppet] setup script fd: %s\n",
strerror(errno));
_exit(126);
}
close(script_fd);
}
if (fcntl(3, F_SETFD, 0) < 0) {
dprintf(STDERR_FILENO, "[sockpuppet] setup script flags: %s\n",
strerror(errno));
_exit(126);
}
#ifdef __NR_close_range
if (syscall(__NR_close_range, 4U, ~0U, 0U) < 0) {
dprintf(STDERR_FILENO, "[sockpuppet] setup descriptor sweep: %s\n",
strerror(errno));
_exit(126);
}
#else
dprintf(STDERR_FILENO, "[sockpuppet] setup descriptor sweep unavailable\n");
_exit(126);
#endif
if (apply_setup_seccomp() < 0) {
dprintf(STDERR_FILENO, "[sockpuppet] setup seccomp: %s\n",
strerror(errno));
_exit(126);
}
snprintf(script_path, sizeof(script_path), "/proc/self/fd/3");
execve("/bin/sh", argv, envp);
dprintf(STDERR_FILENO, "[sockpuppet] setup exec: %s\n", strerror(errno));
_exit(126);
}
static int sp_run_setup_phase(int *script_fd, int stdout_fd, int stderr_fd,
unsigned int timeout_ms) {
int stdin_pipe[2];
pid_t parent = getpid();
pid_t child;
int status = 0;
int timed_out = 0;
uint64_t deadline;
if (script_fd == NULL || *script_fd < 0 || stdout_fd < 0 ||
stderr_fd < 0 || timeout_ms == 0) {
errno = EINVAL;
return 1;
}
if (pipe2(stdin_pipe, O_CLOEXEC) < 0)
return 1;
child = fork();
if (child < 0) {
close(stdin_pipe[0]);
close(stdin_pipe[1]);
return 1;
}
if (child == 0) {
close(stdin_pipe[1]);
sp_setup_exec_child(*script_fd, stdin_pipe[0], stdout_fd, stderr_fd,
parent);
}
close(stdin_pipe[0]);
close(stdin_pipe[1]);
deadline = sp_ns_now_ns() + (uint64_t)timeout_ms * 1000000ULL;
for (;;) {
pid_t got = waitpid(child, &status, WNOHANG);
if (got == child)
break;
if (got < 0) {
if (errno == EINTR)
continue;
(void)kill(child, SIGKILL);
while (waitpid(child, &status, 0) < 0 && errno == EINTR)
;
status = 0;
break;
}
if (sp_ns_now_ns() >= deadline) {
if (kill(child, SIGKILL) < 0 && errno != ESRCH)
status = 0;
while (waitpid(child, &status, 0) < 0 && errno == EINTR)
;
timed_out = 1;
break;
}
(void)poll(NULL, 0, 10);
}
sp_close_owned_fd_nointr(script_fd);
if (kill(-1, SIGKILL) < 0 && errno != ESRCH)
return 1;
for (;;) {
pid_t got = waitpid(-1, NULL, 0);
if (got > 0)
continue;
if (got < 0 && errno == EINTR)
continue;
if (got < 0 && errno == ECHILD)
break;
return 1;
}
if (timed_out)
return 124;
return sp_wait_status_to_exit_code(status);
}
/* ---------- epoll helpers ---------- */
static uint64_t epoll_registration_token(enum fd_type type, size_t index,
int fd, uint32_t generation) {
if (index > UINT8_MAX || fd < 0 || fd > UINT16_MAX || generation == 0)
die("epoll registration token");
return epoll_token_encode(type, (uint8_t)index, (uint16_t)fd, generation);
}
/* Registers (EPOLL_CTL_ADD) or updates (EPOLL_CTL_MOD) one of a UDP flow's
* two descriptors: its relay socket (FD_UDP_RELAY) or SOCKS control
* connection (FD_UDP_CTRL). A failed ADD is fatal; a failed MOD marks the
* flow and the UDP table for teardown. */
static void epoll_set_udp(struct udp_flow *f, int op, enum fd_type kind, uint32_t events) {
int fd = kind == FD_UDP_CTRL ? f->tcp_ctrl : f->udp_relay;
struct epoll_event *interest = kind == FD_UDP_CTRL ? &f->ctrl_interest : &f->relay_interest;
if (fd < 0 || g_epfd < 0)
return;
struct epoll_event ev = {
.events = events,
.data.u64 = epoll_registration_token(kind, (size_t)(f - udp_flows), fd, f->generation)};
if (sp_epoll_update(op, fd, &ev, interest) == 0)
return;
if (op == EPOLL_CTL_ADD)
die(kind == FD_UDP_CTRL ? "epoll_ctl add udp ctrl" : "epoll_ctl add udp");
sp_epoll_update_failed(kind == FD_UDP_CTRL ? "UDP control" : "UDP relay");
f->epoll_failed = 1;
g_udp_epoll_failed = 1;
}
static int epoll_add_publish_rule(struct publish_rule *rule) {
if (rule->listen_fd < 0 || g_epfd < 0)
return 0;
enum fd_type type = rule->proto == IPPROTO_TCP ? FD_PUBLISH_TCP_LISTENER
: FD_PUBLISH_UDP_SOCKET;
struct epoll_event ev = {
.events = EPOLLIN | EPOLLERR | EPOLLHUP,
.data.u64 = epoll_registration_token(type, (size_t)rule->rule_index,
rule->listen_fd, 1),
};
return sp_epoll_update(EPOLL_CTL_ADD, rule->listen_fd, &ev, &rule->interest);
}
static void epoll_del(int fd) {
if (fd >= 0 && g_epfd >= 0)
epoll_ctl(g_epfd, EPOLL_CTL_DEL, fd, NULL);
}
static int set_nonblocking(int fd) {
int flags = fcntl(fd, F_GETFL, 0);
if (flags < 0)
return -1;
return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
}
enum relay_destination_mode {
RELAY_DEST_NONBLOCKING_WRITE = 1,
RELAY_DEST_NONBLOCKING_SEND,
RELAY_DEST_REGULAR_FILE
};
struct relay_destination {
int fd;
enum relay_destination_mode mode;
int owned;
int epollable;
};
struct relay_runtime {
struct sp_relay_state state;
int source_fd;
struct relay_destination destination;
enum fd_type source_type;
enum fd_type destination_type;
int source_registered;
int destination_registered;
struct epoll_event source_interest, destination_interest;
int close_source_on_destination_failure;
};
#ifdef SOCKPUPPET_RELAY_TESTING
static int g_relay_test_read_errno;
static int g_relay_test_write_errno;
static size_t g_relay_test_write_limit;
#endif
static ssize_t relay_os_read(int fd, void *buffer, size_t length) {
#ifdef SOCKPUPPET_RELAY_TESTING
if (g_relay_test_read_errno != 0) {
errno = g_relay_test_read_errno;
g_relay_test_read_errno = 0;
return -1;
}
#endif
return read(fd, buffer, length);
}
static ssize_t relay_os_write(struct relay_runtime *runtime,
const uint8_t *data, size_t length) {
#ifdef SOCKPUPPET_RELAY_TESTING
if (g_relay_test_write_errno != 0) {
errno = g_relay_test_write_errno;
g_relay_test_write_errno = 0;
return -1;
}
if (g_relay_test_write_limit > 0 && length > g_relay_test_write_limit)
length = g_relay_test_write_limit;
#endif
if (runtime->destination.mode == RELAY_DEST_NONBLOCKING_SEND)
return send(runtime->destination.fd, data, length,
MSG_DONTWAIT | MSG_NOSIGNAL);
return write(runtime->destination.fd, data, length);
}
static int relay_probe_epollable(int fd) {
struct epoll_event event = {.events = EPOLLOUT};
if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &event) == 0) {
epoll_del(fd);
return 1;
}
if (errno == EPERM)
return 0;
return -1;
}
static int relay_prepare_destination(int inherited_fd,
struct relay_destination *destination) {
struct stat st;
int flags;
memset(destination, 0, sizeof(*destination));
destination->fd = -1;
if (fstat(inherited_fd, &st) < 0)
return -1;
if (S_ISREG(st.st_mode)) {
destination->fd = inherited_fd;
destination->mode = RELAY_DEST_REGULAR_FILE;
return 0;
}
if (S_ISSOCK(st.st_mode)) {
destination->fd = fcntl(inherited_fd, F_DUPFD_CLOEXEC, 3);
if (destination->fd < 0)
return -1;
destination->mode = RELAY_DEST_NONBLOCKING_SEND;
destination->owned = 1;
destination->epollable = relay_probe_epollable(destination->fd);
if (destination->epollable < 0) {
close(destination->fd);
destination->fd = -1;
destination->owned = 0;
return -1;
}
return 0;
}
char path[64];
int path_len = snprintf(path, sizeof(path), "/proc/self/fd/%d", inherited_fd);
if (path_len < 0 || (size_t)path_len >= sizeof(path)) {
errno = ENAMETOOLONG;
return -1;
}
destination->fd = open(path, O_WRONLY | O_NONBLOCK | O_CLOEXEC);
if (destination->fd < 0) {
flags = fcntl(inherited_fd, F_GETFL, 0);
if (flags < 0 || (flags & O_NONBLOCK) == 0)
return -1;
destination->fd = fcntl(inherited_fd, F_DUPFD_CLOEXEC, 3);
if (destination->fd < 0)
return -1;
}
destination->mode = RELAY_DEST_NONBLOCKING_WRITE;
destination->owned = 1;
destination->epollable = relay_probe_epollable(destination->fd);
if (destination->epollable < 0) {
close(destination->fd);
destination->fd = -1;
destination->owned = 0;
return -1;
}
return 0;
}
static void relay_close_destination(struct relay_destination *destination) {
int fd = destination->owned ? destination->fd : -1;
destination->fd = -1;
destination->owned = 0;
destination->epollable = 0;
if (fd >= 0)
(void)close(fd);
}
static void relay_destination_failed(struct relay_runtime *runtime) {
(void)sp_relay_step(
&runtime->state,
&(struct sp_relay_event){.type = SP_RELAY_DESTINATION_FAILED});
}
static void relay_runtime_init(struct relay_runtime *runtime, int source_fd,
const struct relay_destination *destination,
enum fd_type source_type,
enum fd_type destination_type, uint8_t *storage) {
memset(runtime, 0, sizeof(*runtime));
sp_relay_init(&runtime->state, storage, RELAY_QUEUE_CAP);
runtime->source_fd = source_fd;
runtime->destination = *destination;
runtime->source_type = source_type;
runtime->destination_type = destination_type;
runtime->close_source_on_destination_failure = 1;
}
static void relay_close_source(struct relay_runtime *runtime) {
int fd;
if (runtime->source_fd < 0)
return;
fd = runtime->source_fd;
runtime->source_fd = -1;
runtime->source_registered = 0;
runtime->source_interest = (struct epoll_event){0};
epoll_del(fd);
(void)close(fd);
(void)sp_relay_step(
&runtime->state,
&(struct sp_relay_event){.type = SP_RELAY_SOURCE_EOF});
}
static void relay_drain(struct relay_runtime *runtime, size_t budget) {
while (sp_relay_wants_write(&runtime->state) && budget > 0) {
size_t length;
const uint8_t *data = sp_relay_output(&runtime->state, &length);
if (length > budget)
length = budget;
ssize_t written = relay_os_write(runtime, data, length);
if (written > 0) {
(void)sp_relay_step(
&runtime->state,
&(struct sp_relay_event){.type = SP_RELAY_DESTINATION_WRITTEN,
.length = (size_t)written});
budget -= (size_t)written;
continue;
}
if (written < 0 && errno == EINTR)
continue;
if (written < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
return;
relay_destination_failed(runtime);
if (runtime->close_source_on_destination_failure)
relay_close_source(runtime);
return;
}
}
static void relay_drain_from_batch(struct relay_runtime *runtime,
size_t *batch_budget) {
size_t budget = *batch_budget;
if (budget > RELAY_DRAIN_BUDGET)
budget = RELAY_DRAIN_BUDGET;
uint64_t before = runtime->state.bytes_out;
relay_drain(runtime, budget);
uint64_t drained = runtime->state.bytes_out - before;
if (drained > *batch_budget)
*batch_budget = 0;
else
*batch_budget -= (size_t)drained;
}
/* Reads what fits into the relay. At end of input or on a read error the
* batch relay closes its source; the interactive relay keeps the descriptor
* and only records EOF. */
static void relay_read_source(struct relay_runtime *runtime, int close_at_eof) {
size_t space = sp_relay_space(&runtime->state);
if (space == 0)
return;
if (space > sizeof(g_io_buf))
space = sizeof(g_io_buf);
ssize_t count = relay_os_read(runtime->source_fd, g_io_buf, space);
if (count > 0) {
(void)sp_relay_step(
&runtime->state,
&(struct sp_relay_event){.type = SP_RELAY_SOURCE_BYTES,
.bytes = g_io_buf,
.length = (size_t)count});
return;
}
if (count < 0 && (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK))
return;
if (close_at_eof)
relay_close_source(runtime);
else
(void)sp_relay_step(&runtime->state, &(struct sp_relay_event){.type = SP_RELAY_SOURCE_EOF});
}
static int interactive_open_pty_master(void) {
return posix_openpt(O_RDWR | O_NOCTTY | O_CLOEXEC);
}
static int interactive_sync_winsize(struct interactive_session *session) {
if (!session || !session->active || session->host_tty_fd < 0 ||
session->pty_master_fd < 0)
return 0;
if (ioctl(session->host_tty_fd, TIOCGWINSZ, &session->host_winsize) < 0) {
if (errno == ENOTTY)
return 0;
return -1;
}
if (ioctl(session->pty_master_fd, TIOCSWINSZ, &session->host_winsize) < 0)
return -1;
return 0;
}
static int interactive_restore_terminal(struct interactive_session *session) {
if (!session || session->host_tty_fd < 0 || !session->host_termios_saved)
return 0;
for (;;) {
if (tcsetattr(session->host_tty_fd, TCSAFLUSH, &session->host_termios) == 0) {
g_stderr_raw_tty = 0;
return 0;
}
if (errno != EINTR)
return -1;
}
}
static int interactive_close_session(struct interactive_session *session) {
int restore_failed = 0;
if (!session)
return 0;
if (interactive_restore_terminal(session) < 0) {
restore_failed = 1;
fprintf(stderr, "[sockpuppet] Error: could not restore parent terminal: %s\n",
strerror(errno));
}
sp_close_owned_fd_nointr(&session->pty_master_fd);
sp_close_owned_fd_nointr(&session->pty_slave_fd);
sp_close_owned_fd_nointr(&session->host_tty_fd);
session->active = 0;
return restore_failed ? -1 : 0;
}
static void interactive_atexit_cleanup(void) {
(void)interactive_close_session(&interactive_session);
}
static int interactive_parent_setup(struct interactive_session *session) {
struct termios raw;
memset(session, 0, sizeof(*session));
session->host_tty_fd = -1;
session->pty_master_fd = -1;
session->pty_slave_fd = -1;
session->host_tty_fd = open("/dev/tty", O_RDWR | O_NOCTTY | O_CLOEXEC);
if (session->host_tty_fd < 0)
return -1;
if (!isatty(session->host_tty_fd)) {
errno = ENOTTY;
return -1;
}
session->pty_master_fd = interactive_open_pty_master();
if (session->pty_master_fd < 0)
return -1;
if (grantpt(session->pty_master_fd) < 0)
return -1;
if (unlockpt(session->pty_master_fd) < 0)
return -1;
#ifdef TIOCGPTPEER
session->pty_slave_fd =
ioctl(session->pty_master_fd, TIOCGPTPEER, O_RDWR | O_NOCTTY | O_CLOEXEC);
if (session->pty_slave_fd < 0)
return -1;
#else
errno = ENOTSUP;
return -1;
#endif
if (tcgetattr(session->host_tty_fd, &session->host_termios) < 0)
return -1;
session->host_termios_saved = 1;
raw = session->host_termios;
cfmakeraw(&raw);
if (tcsetattr(session->host_tty_fd, TCSAFLUSH, &raw) < 0)
return -1;
session->active = 1;
if (interactive_sync_winsize(session) < 0)
return -1;
return 0;
}
/* Names the first way fd differs from an AF_INET socket of `type` and
* `protocol` (SOCK_STREAM/IPPROTO_TCP or SOCK_DGRAM/IPPROTO_UDP), or NULL.
* Checked before any option call, so post-hardening SOL_SOCKET authority
* cannot be redirected to another descriptor kind. */
static const char *sp_host_socket_kind_error(int fd, int type, int protocol) {
int value = 0;
socklen_t length = sizeof(value);
int tcp = type == SOCK_STREAM;
if (getsockopt(fd, SOL_SOCKET, SO_DOMAIN, &value, &length) < 0 || value != AF_INET)
return "not an AF_INET socket";
length = sizeof(value);
if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &value, &length) < 0 || value != type)
return tcp ? "not a SOCK_STREAM socket" : "not a SOCK_DGRAM socket";
length = sizeof(value);
if (getsockopt(fd, SOL_SOCKET, SO_PROTOCOL, &value, &length) < 0 || value != protocol)
return tcp ? "not an IPPROTO_TCP socket" : "not an IPPROTO_UDP socket";
return NULL;
}
/* Host TCP socket-option ownership.
*
* Every established direct, SOCKS-ready, and accepted-publish host TCP socket
* is configured here and nowhere else. Concentrating the calls is what lets
* the parent seccomp filter permit exactly three (level, option, length)
* triples instead of granting steady-state socket-option authority; see
* parent_seccomp_append_host_tcp_option_allow().
*
* seccomp cannot inspect what kind of file a descriptor refers to, so the fd
* kind is validated here in userspace before any option is set.
*
* Timing: callers on a connect path must invoke this before connect(), because
* SO_RCVBUF has to be in effect before the SYN for window scale to be
* negotiated from the configured buffer rather than the default. The publish
* path is the unavoidable exception — its fd arrives already established from
* accept4(), so the resize cannot influence the scale that was negotiated.
*
* Returns 0 on success, -1 with *reason set to a static string on failure. */
static int sp_scheduler_configure_host_tcp_socket(int fd, const char **reason) {
static const struct {
int level;
int option;
int value;
const char *name;
} options[] = {
{IPPROTO_TCP, TCP_NODELAY, 1, "TCP_NODELAY"},
{SOL_SOCKET, SO_SNDBUF, SP_HOST_TCP_SNDBUF, "SO_SNDBUF"},
{SOL_SOCKET, SO_RCVBUF, SP_HOST_TCP_RCVBUF, "SO_RCVBUF"},
};
socklen_t length;
size_t i;
*reason = sp_host_socket_kind_error(fd, SOCK_STREAM, IPPROTO_TCP);
if (*reason != NULL)
return -1;
for (i = 0; i < sizeof(options) / sizeof(options[0]); ++i) {
int value = options[i].value;
int effective = 0;
if (setsockopt(fd, options[i].level, options[i].option, &value,
sizeof(value)) < 0) {
*reason = options[i].name;
return -1;
}
length = sizeof(effective);
if (getsockopt(fd, options[i].level, options[i].option, &effective,
&length) < 0 || length != sizeof(effective)) {
*reason = options[i].name;
return -1;
}
if (options[i].option == TCP_NODELAY) {
/* Nagle silently left on would batch every small response, so this is a
* strict gate rather than a logged observation. */
if (effective == 0) {
*reason = "TCP_NODELAY readback was 0";
return -1;
}
continue;
}
if (effective < SP_HOST_TCP_BUF_MIN || effective > SP_HOST_TCP_BUF_MAX) {
*reason = options[i].name;
return -1;
}
DBG("host TCP %s: requested %d effective %d", options[i].name, value,
effective);
}
return 0;
}
/* UDP sockets use the same frozen Linux buffer readback band as TCP, after
* the same descriptor kind check. */
static int sp_udp_configure_host_socket(int fd, int *effective_sndbuf,
int *effective_rcvbuf,
const char **reason) {
int request;
int effective;
socklen_t length;
*reason = sp_host_socket_kind_error(fd, SOCK_DGRAM, IPPROTO_UDP);
if (*reason != NULL)
return -1;
request = SP_HOST_UDP_SNDBUF;
if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &request, sizeof(request)) < 0) {
*reason = "SO_SNDBUF";
return -1;
}
length = sizeof(effective);
if (getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &effective, &length) < 0 ||
length != sizeof(effective) || effective < SP_HOST_TCP_BUF_MIN ||
effective > SP_HOST_TCP_BUF_MAX) {
*reason = "SO_SNDBUF readback";
return -1;
}
*effective_sndbuf = effective;
request = SP_HOST_UDP_RCVBUF;
if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &request, sizeof(request)) < 0) {
*reason = "SO_RCVBUF";
return -1;
}
length = sizeof(effective);
if (getsockopt(fd, SOL_SOCKET, SO_RCVBUF, &effective, &length) < 0 ||
length != sizeof(effective) || effective < SP_HOST_TCP_BUF_MIN ||
effective > SP_HOST_TCP_BUF_MAX) {
*reason = "SO_RCVBUF readback";
return -1;
}
*effective_rcvbuf = effective;
return 0;
}
static int sp_ns_segment_needs_challenge_ack(
const struct sp_tcp_core *core, const struct sp_ev_tcp_segment *segment) {
uint64_t seq_abs;
uint64_t sequence_len;
if (segment == NULL || core->state < SP_TCP_CORE_ESTABLISHED ||
core->state > SP_TCP_CORE_TIME_WAIT)
return 0;
if ((segment->flags & SP_TCP_CORE_SYN) != 0)
return 1;
if ((segment->flags & SP_TCP_CORE_RST) == 0)
return 0;
sequence_len = segment->payload_length.value;
if ((segment->flags & SP_TCP_CORE_FIN) != 0)
sequence_len++;
seq_abs = sp_tcp_unwrap_u32(segment->seq, core->rcv_acked_abs);
return sp_segment_acceptable(core, seq_abs, sequence_len) &&
seq_abs != core->rcv_acked_abs;
}
/* Project the core into what the executor is allowed to see. The engine
* wrapper computes the challenge-ACK decision before the transition, then
* calls this after commit or rollback to capture everything else. */
/* The engine wrapper is the sole owner of observations that need pre-event
* core state but are not protocol effects. RFC 5961 challenge classification
* happens before dispatch; status is captured only after commit or rollback.
* Direct, loopback, and split execution all enter here. */
static int start_nonblocking_connect(int fd, const struct sockaddr_in *addr) {
const char *reason = "unknown";
/* Before connect(): SO_RCVBUF must be in effect for window-scale
* negotiation. */
if (sp_scheduler_configure_host_tcp_socket(fd, &reason) < 0) {
DBG("host TCP option setup failed: %s (%s)", reason, strerror(errno));
return -1;
}
if (set_nonblocking(fd) < 0)
return -1;
if (connect(fd, (const struct sockaddr *)addr, sizeof(*addr)) == 0)
return 0;
if (errno == EINPROGRESS)
return 1;
return -1;
}
static int socket_connect_complete(int fd) {
int so_error = 0;
socklen_t len = sizeof(so_error);
if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &so_error, &len) < 0)
return -1;
if (so_error != 0) {
errno = so_error;
return -1;
}
return 0;
}
/* Sans-I/O SOCKS: all inputs are values, with owned bounded rx/tx storage.
* Socket readiness, errno, clocks, addresses and logging belong to the adapter. */
struct sp_socks_config {
const char *username;
const char *password;
uint32_t proxy_ip; /* network byte order */
};
struct sp_socks_endpoint {
uint32_t ip; /* network byte order */
uint16_t port; /* host byte order */
};
static void socks_io_reset(struct socks_io *io) { memset(io, 0, sizeof(*io)); }
static int socks_has_pending_tx(const struct socks_io *io) {
return io->tx_off < io->tx_len;
}
static size_t socks_response_need(const struct socks_io *io) {
if (io->state == SOCKS_IO_METHOD || io->state == SOCKS_IO_AUTH)
return 2;
if (io->state != SOCKS_IO_REQUEST)
return 0;
if (io->rx_len < 4)
return 4;
if (io->rxbuf[3] == 0x01)
return 10;
if (io->rxbuf[3] == 0x04)
return 22;
if (io->rxbuf[3] == 0x03) {
if (io->rx_len < 5)
return 5;
size_t total = (size_t)(4 + 1 + io->rxbuf[4] + 2);
if (total > sizeof(io->rxbuf))
return sizeof(io->rxbuf) + 1; /* Trigger failure check */
return total;
}
return sizeof(io->rxbuf) + 1; /* Reject an unknown address type. */
}
static void socks_consume_rx(struct socks_io *io, size_t used) {
if (used >= io->rx_len) {
io->rx_len = 0;
return;
}
memmove(io->rxbuf, io->rxbuf + used, io->rx_len - used);
io->rx_len -= used;
}
static int socks_queue_send(struct socks_io *io, const uint8_t *data, size_t len) {
if (len > sizeof(io->txbuf))
return -1;
memcpy(io->txbuf, data, len);
io->tx_off = 0;
io->tx_len = len;
return 0;
}
static int socks_queue_greeting(struct socks_io *io, const struct sp_socks_config *cfg) {
uint8_t greeting[4];
if (cfg->username[0] != '\0') {
greeting[0] = 0x05;
greeting[1] = 0x02;
greeting[2] = 0x00;
greeting[3] = 0x02;
return socks_queue_send(io, greeting, 4);
}
greeting[0] = 0x05;
greeting[1] = 0x01;
greeting[2] = 0x00;
return socks_queue_send(io, greeting, 3);
}
static int socks_queue_auth(struct socks_io *io, const struct sp_socks_config *cfg) {
size_t ulen = strlen(cfg->username);
size_t plen = strlen(cfg->password);
uint8_t auth[513];
size_t off = 0;
if (ulen > 255 || plen > 255)
return -1;
auth[off++] = 0x01;
auth[off++] = (uint8_t)ulen;
memcpy(auth + off, cfg->username, ulen);
off += ulen;
auth[off++] = (uint8_t)plen;
memcpy(auth + off, cfg->password, plen);
off += plen;
return socks_queue_send(io, auth, off);
}
static int socks_queue_request(struct socks_io *io) {
uint8_t req[4 + 1 + 255 + 2];
uint16_t port_be = htons(io->target_port);
size_t off = 0;
size_t domain_len = strlen(io->target_domain);
if (domain_len > 255)
return -1;
req[off++] = 0x05;
req[off++] = io->is_udp ? 0x03 : 0x01;
req[off++] = 0x00;
if (!io->is_udp && domain_len > 0) {
req[off++] = 0x03;
req[off++] = (uint8_t)domain_len;
memcpy(req + off, io->target_domain, domain_len);
off += domain_len;
} else {
req[off++] = 0x01;
memcpy(req + off, &io->target_ip, 4);
off += 4;
}
memcpy(req + off, &port_be, 2);
off += 2;
return socks_queue_send(io, req, off);
}
static int sp_socks_begin_handshake(struct socks_io *io,
const struct sp_socks_config *cfg) {
if (socks_queue_greeting(io, cfg) < 0)
return -1;
io->state = SOCKS_IO_METHOD;
return 0;
}
static int sp_socks_process_rx(struct socks_io *io, const struct sp_socks_config *cfg,
struct sp_socks_endpoint *relay_addr) {
for (;;) {
size_t need = socks_response_need(io);
if (io->state == SOCKS_IO_READY)
return 1;
if (socks_has_pending_tx(io))
return 0;
if (need > sizeof(io->rxbuf)) {
io->state = SOCKS_IO_FAILED;
return -1;
}
if (need == 0 || io->rx_len < need)
return 0;
if (io->state == SOCKS_IO_METHOD) {
if (io->rxbuf[0] != 0x05) {
io->state = SOCKS_IO_FAILED;
return -1;
}
if (io->rxbuf[1] == 0x00) {
socks_consume_rx(io, 2);
if (socks_queue_request(io) < 0) {
io->state = SOCKS_IO_FAILED;
return -1;
}
io->state = SOCKS_IO_REQUEST;
continue;
}
if (io->rxbuf[1] == 0x02 && cfg->username[0] != '\0') {
socks_consume_rx(io, 2);
if (socks_queue_auth(io, cfg) < 0) {
io->state = SOCKS_IO_FAILED;
return -1;
}
io->state = SOCKS_IO_AUTH;
continue;
}
io->state = SOCKS_IO_FAILED;
return -1;
}
if (io->state == SOCKS_IO_AUTH) {
if (io->rxbuf[0] != 0x01 || io->rxbuf[1] != 0x00) {
io->state = SOCKS_IO_FAILED;
return -1;
}
socks_consume_rx(io, 2);
if (socks_queue_request(io) < 0) {
io->state = SOCKS_IO_FAILED;
return -1;
}
io->state = SOCKS_IO_REQUEST;
continue;
}
if (io->state == SOCKS_IO_REQUEST) {
uint8_t atyp;
size_t used = need;
if (io->rxbuf[0] != 0x05 || io->rxbuf[1] != 0x00 ||
io->rxbuf[2] != 0x00) {
io->state = SOCKS_IO_FAILED;
return -1;
}
atyp = io->rxbuf[3];
if (atyp != 0x01 && atyp != 0x03 && atyp != 0x04) {
io->state = SOCKS_IO_FAILED;
return -1;
}
if (relay_addr) {
memset(relay_addr, 0, sizeof(*relay_addr));
if (atyp == 0x01 && need >= 10) {
memcpy(&relay_addr->ip, io->rxbuf + 4, 4);
relay_addr->port = (uint16_t)((io->rxbuf[8] << 8) | io->rxbuf[9]);
} else if (atyp == 0x03 && need >= 7) {
relay_addr->ip = cfg->proxy_ip;
relay_addr->port = (uint16_t)((io->rxbuf[need - 2] << 8) | io->rxbuf[need - 1]);
} else if (atyp == 0x04) {
io->state = SOCKS_IO_FAILED;
return -1;
} else {
io->state = SOCKS_IO_FAILED;
return -1;
}
if (relay_addr->ip == 0 ||
relay_addr->ip == htonl(0x7f000001)) {
relay_addr->ip = cfg->proxy_ip;
}
if (relay_addr->port == 0) {
io->state = SOCKS_IO_FAILED;
return -1;
}
}
socks_consume_rx(io, used);
if (io->is_udp && io->rx_len != 0) {
io->state = SOCKS_IO_FAILED;
return -1;
}
io->state = SOCKS_IO_READY;
return 1;
}
}
}
static int sp_socks_tx_accepted(struct socks_io *io, size_t accepted) {
if (accepted > io->tx_len - io->tx_off)
return -1;
io->tx_off += accepted;
if (io->tx_off == io->tx_len)
io->tx_off = io->tx_len = 0;
return 0;
}
static int sp_socks_receive(struct socks_io *io, const uint8_t *bytes,
size_t length, const struct sp_socks_config *cfg,
struct sp_socks_endpoint *relay) {
if (length > sizeof(io->rxbuf) - io->rx_len)
return -1;
memcpy(io->rxbuf + io->rx_len, bytes, length);
io->rx_len += length;
return sp_socks_process_rx(io, cfg, relay);
}
static int sp_socks_udp_encode(struct sp_socks_endpoint endpoint,
const char *domain, size_t domain_len, const uint8_t *payload,
size_t payload_len, uint8_t *packet, size_t capacity, size_t *packet_len) {
size_t header_len, offset = 0;
if (domain_len > 255)
return -1;
header_len = domain_len != 0 ? 7U + domain_len : 10U;
if (payload_len > capacity || header_len > capacity - payload_len)
return -1;
packet[offset++] = 0;
packet[offset++] = 0;
packet[offset++] = 0;
if (domain_len != 0) {
packet[offset++] = 0x03;
packet[offset++] = (uint8_t)domain_len;
memcpy(packet + offset, domain, domain_len);
offset += domain_len;
} else {
packet[offset++] = 0x01;
memcpy(packet + offset, &endpoint.ip, 4);
offset += 4;
}
packet[offset++] = (uint8_t)(endpoint.port >> 8);
packet[offset++] = (uint8_t)(endpoint.port & 0xff);
memcpy(packet + offset, payload, payload_len);
*packet_len = offset + payload_len;
return 0;
}
static int sp_socks_udp_decode(struct sp_socks_endpoint endpoint,
const char *domain, size_t domain_len,
const uint8_t *packet, size_t packet_len,
const uint8_t **payload,
size_t *payload_len) {
size_t header_len;
uint16_t port;
if (packet_len < 4 || packet[0] != 0 || packet[1] != 0 ||
packet[2] != 0)
return -1;
if (packet[3] == 0x01) {
uint32_t address;
header_len = 10;
if (packet_len < header_len)
return -1;
memcpy(&address, packet + 4, sizeof(address));
port = (uint16_t)(((uint16_t)packet[8] << 8) | packet[9]);
if (address != endpoint.ip || port != endpoint.port)
return -1;
} else if (packet[3] == 0x03) {
size_t name_len;
if (packet_len < 5)
return -1;
name_len = packet[4];
header_len = 7U + name_len;
if (name_len == 0 || packet_len < header_len || domain_len == 0 ||
domain_len != name_len ||
memcmp(packet + 5, domain, name_len) != 0)
return -1;
port = (uint16_t)(((uint16_t)packet[5 + name_len] << 8) |
packet[6 + name_len]);
if (port != endpoint.port)
return -1;
} else {
return -1;
}
*payload = packet + header_len;
*payload_len = packet_len - header_len;
return 0;
}
/* End Sans-I/O SOCKS. */
static struct sp_socks_config socks_core_config(const struct socks_config *cfg) {
return (struct sp_socks_config){cfg->username, cfg->password,
cfg->addr.sin_addr.s_addr};
}
static int socks_begin_handshake(struct socks_io *io,
const struct socks_config *cfg) {
struct sp_socks_config config = socks_core_config(cfg);
return sp_socks_begin_handshake(io, &config);
}
static int socks_process_rx(struct socks_io *io, const struct socks_config *cfg,
struct sockaddr_in *relay_addr) {
struct sp_socks_config config = socks_core_config(cfg);
struct sp_socks_endpoint relay = {0};
int result = sp_socks_process_rx(io, &config, relay_addr ? &relay : NULL);
if (result == 1 && relay_addr) {
*relay_addr = (struct sockaddr_in){.sin_family = AF_INET,
.sin_addr.s_addr = relay.ip, .sin_port = htons(relay.port)};
}
return result;
}
static int socks_flush_tx(int fd, struct socks_io *io) {
uint64_t started_ns = sp_ns_now_ns();
while (socks_has_pending_tx(io)) {
ssize_t sent =
send(fd, io->txbuf + io->tx_off, io->tx_len - io->tx_off, MSG_NOSIGNAL);
if (sent > 0) {
if (sp_socks_tx_accepted(io, (size_t)sent) < 0)
return -1;
if (sp_ns_now_ns() - started_ns >= SP_CALLBACK_BUDGET_NS)
return 0;
continue;
}
if (sent < 0 && errno == EINTR &&
sp_ns_now_ns() - started_ns < SP_CALLBACK_BUDGET_NS)
continue;
if (sent < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
return 0;
return -1;
}
return 0;
}
static int socks_recv_and_process(int fd, struct socks_io *io,
const struct socks_config *cfg,
struct sockaddr_in *relay_addr) {
uint64_t started_ns = sp_ns_now_ns();
for (;;) {
uint8_t bytes[sizeof(io->rxbuf)];
int parsed = socks_process_rx(io, cfg, relay_addr);
if (parsed != 0)
return parsed;
if (socks_has_pending_tx(io))
return 0;
ssize_t r = recv(fd, bytes, sizeof(bytes) - io->rx_len, 0);
if (r > 0) {
struct sp_socks_config config = socks_core_config(cfg);
struct sp_socks_endpoint relay = {0};
int result = sp_socks_receive(io, bytes, (size_t)r, &config,
relay_addr ? &relay : NULL);
if (result == 1 && relay_addr)
*relay_addr = (struct sockaddr_in){.sin_family = AF_INET,
.sin_addr.s_addr = relay.ip, .sin_port = htons(relay.port)};
if (result != 0)
return result;
if (socks_has_pending_tx(io))
return 0;
if (sp_ns_now_ns() - started_ns >= SP_CALLBACK_BUDGET_NS)
return 0;
continue;
}
if (r == 0)
return -1;
if (errno == EINTR &&
sp_ns_now_ns() - started_ns < SP_CALLBACK_BUDGET_NS)
continue;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return 0;
return -1;
}
}
enum {
SP_CAPABILITY_V3_MAX = _LINUX_CAPABILITY_U32S_3 * 32 - 1,
};
static int sp_runtime_last_capability(int *last_capability) {
char value[32];
long parsed;
if (read_text_file_trimmed("/proc/sys/kernel/cap_last_cap", value,
sizeof(value)) == 0) {
if (parse_long_strict(value, 0, LONG_MAX, &parsed) < 0) {
if (errno == 0)
errno = EINVAL;
return -1;
}
if (parsed > SP_CAPABILITY_V3_MAX) {
errno = EOVERFLOW;
return -1;
}
*last_capability = (int)parsed;
return 0;
}
for (int cap = 0; cap <= SP_CAPABILITY_V3_MAX; ++cap) {
int rc = prctl(PR_CAPBSET_READ, cap, 0, 0, 0);
if (rc >= 0)
continue;
if (errno == EINVAL && cap > 0) {
*last_capability = cap - 1;
return 0;
}
return -1;
}
{
int rc = prctl(PR_CAPBSET_READ, SP_CAPABILITY_V3_MAX + 1, 0, 0, 0);
if (rc >= 0) {
errno = EOVERFLOW;
return -1;
}
if (errno != EINVAL)
return -1;
}
*last_capability = SP_CAPABILITY_V3_MAX;
return 0;
}
static int sp_drop_capability_bounding(void) {
int last_capability;
if (sp_runtime_last_capability(&last_capability) < 0)
return -1;
for (int cap = 0; cap <= last_capability; cap++) {
if (prctl(PR_CAPBSET_DROP, cap, 0, 0, 0) < 0)
return -1;
}
return 0;
}
static int sp_clear_capability_sets(void) {
struct __user_cap_header_struct hdr = {
.version = _LINUX_CAPABILITY_VERSION_3,
.pid = 0,
};
struct __user_cap_data_struct data[2] = {{0}};
struct __user_cap_data_struct verify[2] = {{0}};
if (syscall(SYS_capset, &hdr, data) < 0) {
return -1;
}
if (syscall(SYS_capget, &hdr, verify) < 0) {
return -1;
}
if (verify[0].effective || verify[0].permitted || verify[0].inheritable ||
verify[1].effective || verify[1].permitted || verify[1].inheritable) {
errno = EPERM;
return -1;
}
return 0;
}
static int sp_clear_ambient_capabilities(void) {
if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) < 0 &&
errno != EINVAL)
return -1;
return 0;
}
/* Drop all capabilities (for rootless mode). */
static void drop_caps(void) {
if (sp_drop_capability_bounding() < 0 ||
sp_clear_ambient_capabilities() < 0 ||
sp_clear_capability_sets() < 0)
die("drop capabilities");
/* Disable core dumps (prevents leaking sensitive data) */
if (prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) < 0) {
die("PR_SET_DUMPABLE");
}
}
static int sp_install_payload_identity(uid_t uid, gid_t gid) {
const unsigned int securebits =
SECBIT_NOROOT | SECBIT_NOROOT_LOCKED | SECBIT_NO_SETUID_FIXUP |
SECBIT_NO_SETUID_FIXUP_LOCKED | SECBIT_KEEP_CAPS_LOCKED |
SECBIT_NO_CAP_AMBIENT_RAISE | SECBIT_NO_CAP_AMBIENT_RAISE_LOCKED;
uid_t real_uid, effective_uid, saved_uid;
gid_t real_gid, effective_gid, saved_gid;
if (sp_drop_capability_bounding() < 0 ||
sp_clear_ambient_capabilities() < 0 ||
prctl(PR_SET_SECUREBITS, securebits, 0, 0, 0) < 0 ||
/* Set this before the credential drop so the following unprivileged
* Landlock installation cannot depend on a post-drop prctl. */
ensure_no_new_privs() < 0 || setgroups(0, NULL) < 0 ||
setresgid(gid, gid, gid) < 0 ||
setresuid(uid, uid, uid) < 0 || sp_clear_capability_sets() < 0 ||
prctl(PR_SET_DUMPABLE, 0, 0, 0, 0) < 0)
return -1;
if (getresuid(&real_uid, &effective_uid, &saved_uid) < 0 ||
getresgid(&real_gid, &effective_gid, &saved_gid) < 0 ||
real_uid != uid || effective_uid != uid || saved_uid != uid ||
real_gid != gid || effective_gid != gid || saved_gid != gid ||
getgroups(0, NULL) != 0 ||
prctl(PR_GET_SECUREBITS, 0, 0, 0, 0) != (int)securebits ||
prctl(PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) != 1) {
errno = EPERM;
return -1;
}
return 0;
}
static int socks_hex_digit(int c) {
if (c >= '0' && c <= '9')
return c - '0';
c |= 0x20;
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
return -1;
}
/* Percent-decode one URL userinfo field (RFC 3986). RFC 1929 fields are 1-255
* bytes; a decoded NUL is rejected because the fields are C strings. */
static int socks_decode_userinfo(const char *in, size_t len, char *out,
size_t out_size) {
size_t used = 0;
for (size_t i = 0; i < len; ++i) {
int c = (unsigned char)in[i];
if (c == '%') {
int high = len - i >= 3 ? socks_hex_digit(in[i + 1]) : -1;
int low = high >= 0 ? socks_hex_digit(in[i + 2]) : -1;
if (low < 0)
return -1;
c = high * 16 + low;
i += 2;
}
if (c == 0 || used + 1 >= out_size)
return -1;
out[used++] = (char)c;
}
out[used] = '\0';
return used == 0 ? -1 : 0;
}
/* Parse SOCKS URL: [socks5://|socks5h://][user:password@]host[:port].
* Error messages never echo the URL because it may carry credentials. */
static int parse_socks_url(const char *url, struct socks_config *cfg,
char *err, size_t err_len) {
const char *p = url;
const char *colon;
const char *at;
size_t hlen;
long port = 1080; /* default SOCKS port */
memset(cfg, 0, sizeof(*cfg));
cfg->remote_dns = 1; /* bare host:port defaults to socks5h */
/* Skip protocol prefix if present */
if (strncmp(p, "socks5h://", 10) == 0) {
cfg->remote_dns = 1;
p += 10;
} else if (strncmp(p, "socks5://", 9) == 0) {
cfg->remote_dns = 0;
p += 9;
} else if (strncmp(p, "socks://", 8) == 0) {
cfg->remote_dns = 0;
p += 8;
}
/* The last '@' ends the userinfo, so a raw '@' may appear in the password;
* the first ':' separates the fields, so a username needs %3A. */
at = strrchr(p, '@');
if (at != NULL) {
colon = memchr(p, ':', (size_t)(at - p));
if (colon == NULL ||
socks_decode_userinfo(p, (size_t)(colon - p), cfg->username,
sizeof(cfg->username)) < 0 ||
socks_decode_userinfo(colon + 1, (size_t)(at - colon - 1),
cfg->password, sizeof(cfg->password)) < 0) {
memset(cfg, 0, sizeof(*cfg));
snprintf(err, err_len, "credentials must be user:password, each 1-255 "
"bytes after percent-decoding");
return -1;
}
p = at + 1;
}
colon = strrchr(p, ':');
hlen = colon != NULL ? (size_t)(colon - p) : strlen(p);
if (hlen == 0 || hlen >= sizeof(cfg->host) ||
(colon != NULL && parse_long_strict(colon + 1, 1, 65535, &port) < 0)) {
memset(cfg, 0, sizeof(*cfg));
snprintf(err, err_len, "expected host[:port] with port 1-65535");
return -1;
}
memcpy(cfg->host, p, hlen);
cfg->host[hlen] = '\0';
cfg->port = (int)port;
cfg->enabled = 1;
return 0;
}
static int add_host_rule_from_spec(const char *spec, char *err,
size_t err_len) {
char buf[128];
char *colon;
char *slash;
long last_octet;
long port;
struct host_rule rule = {0};
int copied;
if (host_rule_count >= MAX_HOST_RULES) {
snprintf(err, err_len, "too many --allow-host rules");
return -1;
}
copied = snprintf(buf, sizeof(buf), "%s", spec);
if (copied < 0 || copied >= (int)sizeof(buf)) {
snprintf(err, err_len, "--allow-host rule too long");
return -1;
}
slash = strchr(buf, '/');
if (slash != NULL) {
*slash = '\0';
if (strcmp(slash + 1, "tcp") == 0)
rule.proto = IPPROTO_TCP;
else if (strcmp(slash + 1, "udp") == 0)
rule.proto = IPPROTO_UDP;
else {
snprintf(err, err_len, "invalid --allow-host protocol");
return -1;
}
}
if (strncmp(buf, "127.0.0.", 8) != 0) {
snprintf(err, err_len, "--allow-host address must be 127.0.0.X");
return -1;
}
colon = strchr(buf + 8, ':');
if (colon == NULL) {
snprintf(err, err_len, "--allow-host must be 127.0.0.X:PORT[/tcp|/udp]");
return -1;
}
*colon = '\0';
if (parse_long_strict(buf + 8, 0, 255, &last_octet) < 0) {
snprintf(err, err_len, "invalid --allow-host address");
return -1;
}
if (strcmp(colon + 1, "*") == 0) {
snprintf(err, err_len, "--allow-host does not allow wildcard ports");
return -1;
}
if (parse_long_strict(colon + 1, 1, 65535, &port) < 0) {
snprintf(err, err_len, "invalid --allow-host port");
return -1;
}
rule.last_octet = (uint8_t)last_octet;
rule.port = (uint16_t)port;
host_rules[host_rule_count++] = rule;
return 0;
}
static int parse_direct_allow_spec(const char *spec, char *err,
size_t err_len) {
char buf[128];
char *colon;
char *proto_slash;
char *prefix_slash;
const char *addr_str;
const char *port_str;
long port;
long prefix = 32;
struct in_addr addr;
struct direct_rule rule = {0};
if (direct_rule_count >= MAX_DIRECT_RULES) {
snprintf(err, err_len, "too many --allow-direct rules");
return -1;
}
if (snprintf(buf, sizeof(buf), "%s", spec) >= (int)sizeof(buf)) {
snprintf(err, err_len, "--allow-direct rule too long");
return -1;
}
colon = strrchr(buf, ':');
if (colon == NULL) {
snprintf(err, err_len,
"--allow-direct must be ADDR[/PREFIX]:PORT[/tcp|/udp]");
return -1;
}
proto_slash = strchr(colon + 1, '/');
if (proto_slash != NULL) {
*proto_slash = '\0';
if (strcmp(proto_slash + 1, "tcp") == 0)
rule.proto = IPPROTO_TCP;
else if (strcmp(proto_slash + 1, "udp") == 0)
rule.proto = IPPROTO_UDP;
else {
snprintf(err, err_len, "invalid --allow-direct protocol");
return -1;
}
}
*colon = '\0';
addr_str = buf;
port_str = colon + 1;
if (parse_long_strict(port_str, 1, 65535, &port) < 0) {
snprintf(err, err_len, "invalid --allow-direct port");
return -1;
}
rule.port = (uint16_t)port;
prefix_slash = strchr(buf, '/');
if (prefix_slash != NULL) {
*prefix_slash = '\0';
if (parse_long_strict(prefix_slash + 1, 0, 32, &prefix) < 0) {
snprintf(err, err_len, "invalid --allow-direct prefix");
return -1;
}
}
if (inet_pton(AF_INET, addr_str, &addr) != 1) {
snprintf(err, err_len, "invalid --allow-direct IPv4 address");
return -1;
}
rule.mask = cidr_mask_from_prefix((unsigned int)prefix);
rule.network = ntohl(addr.s_addr) & rule.mask;
direct_rules[direct_rule_count++] = rule;
return 0;
}
static const char *proto_name(int proto) {
if (proto == IPPROTO_TCP)
return "tcp";
if (proto == IPPROTO_UDP)
return "udp";
return "ip";
}
static int is_loopback_publish_addr(uint32_t ip_be) {
uint32_t ip = ntohl(ip_be);
if ((ip & 0xff000000U) != 0x7f000000U)
return 0;
if (ip == 0x7f000000U || ip == 0x7f000001U || ip == 0x7fffffffU)
return 0;
return 1;
}
static int publish_rule_conflicts(const struct publish_rule *rule) {
for (int i = 0; i < publish_rule_count; ++i) {
const struct publish_rule *existing = &publish_rules[i];
if (existing->host_ip == rule->host_ip &&
existing->host_port == rule->host_port &&
existing->proto == rule->proto)
return 1;
}
return 0;
}
static void publish_rule_debug_string(const struct publish_rule *rule,
char *buf, size_t buf_len) {
char host[INET_ADDRSTRLEN];
char child[INET_ADDRSTRLEN];
struct in_addr host_addr = {.s_addr = rule->host_ip};
struct in_addr child_addr = {.s_addr = SANDBOX_IP};
if (inet_ntop(AF_INET, &host_addr, host, sizeof(host)) == NULL)
snprintf(host, sizeof(host), "unknown");
if (inet_ntop(AF_INET, &child_addr, child, sizeof(child)) == NULL)
snprintf(child, sizeof(child), "unknown");
snprintf(buf, buf_len, "%s %s:%u -> %s:%u", proto_name(rule->proto), host,
rule->host_port, child, rule->container_port);
}
static int parse_publish_spec(const char *spec, struct publish_rule *out,
char *err, size_t err_len) {
char buf[160];
char *slash;
char *first_colon;
char *second_colon;
const char *host_str;
const char *host_port_str;
const char *container_port_str;
long host_port;
long container_port;
struct in_addr host_addr;
memset(out, 0, sizeof(*out));
out->listen_fd = -1;
if (spec == NULL || *spec == '\0') {
snprintf(err, err_len, "empty publish rule");
return -1;
}
if (snprintf(buf, sizeof(buf), "%s", spec) >= (int)sizeof(buf)) {
snprintf(err, err_len, "publish rule too long");
return -1;
}
slash = strchr(buf, '/');
if (slash == NULL) {
snprintf(err, err_len, "publish rule requires /tcp or /udp");
return -1;
}
if (strchr(slash + 1, '/') != NULL) {
snprintf(err, err_len, "publish rule has multiple protocol separators");
return -1;
}
*slash = '\0';
if (strcmp(slash + 1, "tcp") == 0) {
out->proto = IPPROTO_TCP;
} else if (strcmp(slash + 1, "udp") == 0) {
out->proto = IPPROTO_UDP;
} else {
snprintf(err, err_len, "invalid publish protocol");
return -1;
}
first_colon = strchr(buf, ':');
if (first_colon == NULL) {
snprintf(err, err_len,
"--publish must be HOST_LOOPBACK_IP:HOST_PORT:CONTAINER_PORT/PROTO");
return -1;
}
second_colon = strchr(first_colon + 1, ':');
if (second_colon == NULL || strchr(second_colon + 1, ':') != NULL) {
snprintf(err, err_len,
"--publish must contain exactly host IP, host port, and container port");
return -1;
}
*first_colon = '\0';
*second_colon = '\0';
host_str = buf;
host_port_str = first_colon + 1;
container_port_str = second_colon + 1;
if (*host_str == '\0' || *host_port_str == '\0' ||
*container_port_str == '\0') {
snprintf(err, err_len, "publish rule contains an empty field");
return -1;
}
if (inet_pton(AF_INET, host_str, &host_addr) != 1) {
snprintf(err, err_len, "publish host must be an IPv4 address");
return -1;
}
if (!is_loopback_publish_addr(host_addr.s_addr)) {
snprintf(err, err_len,
"publish host must be a non-default 127.0.0.0/8 loopback alias");
return -1;
}
if (parse_long_strict(host_port_str, 1, 65535, &host_port) < 0) {
snprintf(err, err_len, "invalid publish host port");
return -1;
}
if (parse_long_strict(container_port_str, 1, 65535, &container_port) < 0) {
snprintf(err, err_len, "invalid publish container port");
return -1;
}
out->host_ip = host_addr.s_addr;
out->host_port = (uint16_t)host_port;
out->container_port = (uint16_t)container_port;
return 0;
}
static int add_publish_rule_from_spec(const char *spec, char *err,
size_t err_len) {
struct publish_rule rule;
if (publish_rule_count >= MAX_PUBLISH_RULES) {
snprintf(err, err_len, "too many publish rules");
return -1;
}
if (parse_publish_spec(spec, &rule, err, err_len) < 0)
return -1;
if (publish_rule_conflicts(&rule)) {
snprintf(err, err_len, "duplicate published host IP, port, and protocol");
return -1;
}
rule.rule_index = publish_rule_count;
publish_rules[publish_rule_count++] = rule;
return 0;
}
static int parse_egress_mode(const char *value, char *err, size_t err_len) {
if (strcmp(value, "direct") == 0) {
egress_mode = EGRESS_DIRECT;
} else if (strcmp(value, "socks") == 0) {
egress_mode = EGRESS_SOCKS;
} else if (strcmp(value, "none") == 0) {
egress_mode = EGRESS_NONE;
} else {
snprintf(err, err_len, "--egress must be direct, socks, or none");
return -1;
}
egress_mode_explicit = 1;
return 0;
}
static int setup_one_publish_listener(struct publish_rule *rule) {
int type = rule->proto == IPPROTO_TCP ? SOCK_STREAM : SOCK_DGRAM;
int fd = socket(AF_INET, type | SOCK_CLOEXEC, 0);
int one = 1;
struct sockaddr_in addr;
char desc[128];
const char *option_reason = "unknown";
publish_rule_debug_string(rule, desc, sizeof(desc));
if (fd < 0) {
fprintf(stderr, "sockpuppet: failed to publish %s: %s\n", desc,
strerror(errno));
return -1;
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) < 0) {
fprintf(stderr, "sockpuppet: failed to publish %s: SO_REUSEADDR: %s\n",
desc, strerror(errno));
close(fd);
return -1;
}
if (rule->proto == IPPROTO_UDP) {
int pktinfo = 0;
socklen_t pktinfo_len = sizeof(pktinfo);
if (sp_udp_configure_host_socket(
fd, &rule->effective_sndbuf, &rule->effective_rcvbuf,
&option_reason) < 0 ||
setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &one, sizeof(one)) < 0 ||
getsockopt(fd, IPPROTO_IP, IP_PKTINFO, &pktinfo, &pktinfo_len) < 0 ||
pktinfo_len != sizeof(pktinfo) || pktinfo != 1) {
fprintf(stderr,
"sockpuppet: failed to publish %s: UDP options (%s): %s\n",
desc, option_reason, strerror(errno));
close(fd);
return -1;
}
}
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(rule->host_port);
addr.sin_addr.s_addr = rule->host_ip;
if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
fprintf(stderr, "sockpuppet: failed to publish %s: %s\n", desc,
strerror(errno));
close(fd);
return -1;
}
if (rule->proto == IPPROTO_TCP && listen(fd, SOMAXCONN) < 0) {
fprintf(stderr, "sockpuppet: failed to publish %s: listen: %s\n", desc,
strerror(errno));
close(fd);
return -1;
}
if (set_nonblocking(fd) < 0) {
fprintf(stderr, "sockpuppet: failed to publish %s: nonblocking: %s\n",
desc, strerror(errno));
close(fd);
return -1;
}
rule->listen_fd = fd;
DBG("publish %s", desc);
return 0;
}
static void close_publish_listeners(void);
static int setup_publish_listeners(void) {
for (int i = 0; i < publish_rule_count; ++i) {
if (setup_one_publish_listener(&publish_rules[i]) < 0) {
int saved = errno;
close_publish_listeners();
errno = saved;
return -1;
}
}
return 0;
}
static void close_publish_listeners(void) {
for (int i = 0; i < publish_rule_count; ++i) {
if (publish_rules[i].listen_fd >= 0) {
int fd = sp_take_fd(&publish_rules[i].listen_fd);
epoll_del(fd);
(void)close(fd);
}
}
}
static int resolve_socks_proxy(struct socks_config *cfg) {
struct sockaddr_in proxy_addr = {
.sin_family = AF_INET,
.sin_port = htons((uint16_t)cfg->port),
};
if (!cfg->enabled)
return 0;
if (cfg->addr_valid)
return 0;
if (inet_pton(AF_INET, cfg->host, &proxy_addr.sin_addr) <= 0) {
struct addrinfo hints = {.ai_family = AF_INET, .ai_socktype = SOCK_STREAM};
struct addrinfo *res;
if (getaddrinfo(cfg->host, NULL, &hints, &res) != 0)
return -1;
proxy_addr.sin_addr = ((struct sockaddr_in *)res->ai_addr)->sin_addr;
freeaddrinfo(res);
}
cfg->addr = proxy_addr;
cfg->addr_valid = 1;
return 0;
}
extern char **environ;
/* close_range is required: the payload spawn and the setup helpers use it
* without a fallback, so a scan of /proc/self/fd could never help a launch. */
/* Close every fd from 3 up except the bench trace fd and also_keep (-1:
* none). */
static void close_extra_fds_for_exec(int also_keep) {
int keep[2] = {bench_trace_preserved_fd(), also_keep};
unsigned int next = 3;
if (keep[0] > keep[1]) {
keep[0] = keep[1];
keep[1] = bench_trace_preserved_fd();
}
for (int i = 0; i < 2; ++i) {
if (keep[i] < (int)next)
continue;
if (keep[i] > (int)next &&
syscall(SYS_close_range, next, (unsigned int)keep[i] - 1U, 0U) < 0)
die("close_range");
next = (unsigned int)keep[i] + 1U;
}
if (syscall(SYS_close_range, next, ~0U, 0U) < 0)
die("close_range");
}
static char *make_env_entry(const char *name, const char *value) {
size_t name_len = strlen(name);
size_t value_len = strlen(value);
char *entry = malloc(name_len + 1 + value_len + 1);
if (entry == NULL)
die("malloc env");
memcpy(entry, name, name_len);
entry[name_len] = '=';
memcpy(entry + name_len + 1, value, value_len + 1);
return entry;
}
static char **build_sanitized_envp(const char *cwd) {
static const char default_path[] =
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
char **envp = calloc(6, sizeof(*envp));
if (envp == NULL)
die("calloc envp");
envp[0] = make_env_entry("PATH", default_path);
envp[1] = make_env_entry("HOME", "/tmp/home");
envp[2] = make_env_entry("TMPDIR", "/tmp");
envp[3] = make_env_entry("PWD", cwd);
envp[4] = make_env_entry("LC_ALL", "C");
mkdir("/tmp/home", 0700);
return envp;
}
static volatile sig_atomic_t sp_init_pending_signal = 0;
static const int sp_init_forwarded_signals[] = {
SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGUSR2, SIGALRM, SIGCONT,
#ifdef SIGWINCH
SIGWINCH,
#endif
#ifdef SIGTSTP
SIGTSTP,
#endif
0,
};
static void sp_init_signal_handler(int signo) {
sp_init_pending_signal = signo;
}
static void sp_init_set_signal_handlers(void (*handler)(int)) {
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sigemptyset(&sa.sa_mask);
sa.sa_handler = handler;
for (size_t i = 0; sp_init_forwarded_signals[i] != 0; ++i) {
if (sigaction(sp_init_forwarded_signals[i], &sa, NULL) < 0)
die("sigaction init");
}
}
static void sp_init_install_signal_handlers(void) {
sp_init_set_signal_handlers(sp_init_signal_handler);
}
static void sp_init_forward_pending_signal(pid_t payload_pid) {
int signo = sp_init_pending_signal;
if (signo == 0 || payload_pid <= 0)
return;
sp_init_pending_signal = 0;
if (kill(-payload_pid, signo) < 0 && errno == ESRCH)
(void)kill(payload_pid, signo);
}
static void close_payload_setup_fds(const int stdout_pipe[2],
const int stderr_pipe[2],
struct interactive_session *session) {
if (!interactive_stdio) {
close(stdout_pipe[0]);
close(stdout_pipe[1]);
close(stderr_pipe[0]);
close(stderr_pipe[1]);
}
if (session->pty_slave_fd >= 0) {
close(session->pty_slave_fd);
session->pty_slave_fd = -1;
}
}
/* Payload spawn: clone(CLONE_VM|CLONE_VFORK) with a callback on its own
* stack. Contract: pid1 is single-threaded; the child has its own fd and
* signal tables (no CLONE_FILES or CLONE_SIGHAND) but shares pid1's memory
* until execve or _exit, so it makes only system calls and lock-free libc
* calls. The stack has a guard page. See experiments/clone_spawn.c. */
#if SP_AUDIT_ARCH == 0
#error "payload spawn: the raw sigaction layout is checked only on x86_64, aarch64 and riscv64"
#endif
struct sp_spawn_req {
char *const *argv;
char *const *envp;
int stdio[3]; /* all above fd 2 */
int ctty_fd;
uint64_t mask; /* the payload's mask: pid1's mask before the spawn */
/* Written by the child on failure: the die() message and errno. */
const char *volatile fail_what;
volatile int fail_errno;
};
static int sp_spawn_sigmask(const uint64_t *set, uint64_t *old) {
return (int)syscall(SYS_rt_sigprocmask, SIG_SETMASK, set, old,
sizeof(uint64_t));
}
static void sp_spawn_fail(struct sp_spawn_req *req, const char *what) {
req->fail_errno = errno;
req->fail_what = what;
_exit(127);
}
static int sp_spawn_child(void *arg) {
struct sp_spawn_req *req = arg;
/* The kernel's struct starts with the handler on the supported ABIs; an
* all-zero struct is SIG_DFL without flags. */
const unsigned long kernel_dfl[4] = {0, 0, 0, 0};
/* Reset every caught handler: it would run in shared memory once the mask
* is restored. SIG_IGN runs no code and stays, as with fork. Query and
* reset use the system call, not sigaction(): musl's wrapper takes its
* abort lock for SIGABRT, and glibc's refuses its reserved signals. */
for (int sig = 1; sig < NSIG; ++sig) {
unsigned long cur[4] = {0, 0, 0, 0};
if (sig == SIGKILL || sig == SIGSTOP)
continue;
if (syscall(SYS_rt_sigaction, sig, NULL, cur, sizeof(uint64_t)) < 0)
sp_spawn_fail(req, "sigaction init");
if (cur[0] == (uintptr_t)SIG_DFL || cur[0] == (uintptr_t)SIG_IGN)
continue;
if (syscall(SYS_rt_sigaction, sig, kernel_dfl, NULL, sizeof(uint64_t)) < 0)
sp_spawn_fail(req, "sigaction init");
}
/* A new session: in batch mode the payload has no controlling terminal,
* so it cannot reach the host tty through /dev/tty or tcsetpgrp(). It is
* also a process-group leader, which pid1's kill(-pid, sig) forwarding
* needs. */
if (setsid() < 0)
sp_spawn_fail(req, "setsid payload");
if (req->ctty_fd >= 0 && ioctl(req->ctty_fd, TIOCSCTTY, 0) < 0)
sp_spawn_fail(req, "TIOCSCTTY");
for (int fd = 0; fd < 3; ++fd)
if (dup2(req->stdio[fd], fd) < 0)
sp_spawn_fail(req, "dup2 payload stdio");
/* Close, not CLOEXEC: no inherited fd may be reachable through
* /proc/self/fd while execvpe() resolves the program. */
if (syscall(SYS_close_range, 3U, ~0U, 0U) < 0)
sp_spawn_fail(req, "close_range");
/* Limits and no_new_privs are inherited from pid1. */
if (sp_spawn_sigmask(&req->mask, NULL) < 0)
sp_spawn_fail(req, "sigprocmask");
/* execvpe reads PATH from environ, which pid1 pointed at envp before
* the clone. Its stack use is bounded except glibc's /bin/sh fallback,
* which pid1 sized the stack for. */
execvpe(req->argv[0], req->argv, req->envp);
sp_spawn_fail(req, "exec");
return 0;
}
static pid_t sp_clone_spawn_payload(char **argv, char **envp,
struct interactive_session *session,
const int stdout_pipe[2],
const int stderr_pipe[2]) {
const size_t guard = (size_t)sysconf(_SC_PAGESIZE);
size_t argc = 0, stack_size;
char **saved_environ = environ;
const uint64_t all = ~(uint64_t)0;
struct sp_spawn_req req = {
.argv = argv, .envp = envp, .stdio = {-1, -1, -1}, .ctty_fd = -1};
int null_fd = -1;
pid_t pid;
char *stack;
if (interactive_stdio) {
if (!session || session->pty_slave_fd < 0) {
errno = EBADF;
die("interactive child setup");
}
req.ctty_fd = session->pty_slave_fd;
req.stdio[0] = req.stdio[1] = req.stdio[2] = session->pty_slave_fd;
} else {
null_fd = open("/dev/null", O_RDONLY | O_CLOEXEC);
if (null_fd < 0)
die("open /dev/null for payload stdin");
req.stdio[0] = null_fd;
req.stdio[1] = stdout_pipe[1];
req.stdio[2] = stderr_pipe[1];
}
/* main() keeps fds 0-2 open, so every source is above 2, and no dup2()
* in the child can replace a later source. */
for (int fd = 0; fd < 3; ++fd)
if (req.stdio[fd] <= 2) {
errno = EBADF;
die("payload stdio source in fd 0-2");
}
/* The child inherits pid1's limits. The broker already set no core dumps
* and no locked memory, hard limits included, so only the open-file
* limit changes here, and pid1 uses few fds. */
if (set_nofile_rlimit_checked(RLIMIT_CHILD_NOFILE) < 0)
die("child rlimit");
/* glibc's execvpe() runs a file without an executable header through
* /bin/sh, with argc + 2 pointers on this stack (posix/execvpe.c); 64 KiB
* covers the rest. argv is a suffix of main()'s argv, so the kernel's
* argument limit bounds argc. 16-byte top. */
while (argv[argc] != NULL)
++argc;
stack_size = ((argc + 3) * sizeof(char *) + 64 * 1024 + 15) & ~(size_t)15;
/* The child's execvpe() searches the PATH in environ. pid1 makes this
* write itself, so the child does not change shared memory, and restores
* environ once the child has stopped sharing memory. */
environ = envp;
stack = mmap(NULL, stack_size + guard, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
if (stack == MAP_FAILED)
die("mmap spawn stack");
if (mprotect(stack, guard, PROT_NONE) < 0)
die("mprotect spawn stack");
if (sp_spawn_sigmask(&all, &req.mask) < 0)
die("block signals for spawn");
pid = clone(sp_spawn_child, stack + guard + stack_size,
CLONE_VM | CLONE_VFORK | SIGCHLD, &req);
if (pid < 0)
die("fork payload");
environ = saved_environ;
if (sp_spawn_sigmask(&req.mask, NULL) < 0)
die("restore signal mask after spawn");
munmap(stack, stack_size + guard);
if (null_fd >= 0)
close(null_fd);
if (req.fail_what != NULL) {
while (waitpid(pid, NULL, 0) < 0 && errno == EINTR)
;
errno = req.fail_errno;
die(req.fail_what);
}
return pid;
}
static int run_payload_init(char **argv, char **envp,
struct interactive_session *session,
const int stdout_pipe[2],
const int stderr_pipe[2], int report_fd) {
sp_init_install_signal_handlers();
bench_phase_begin("payload_fork");
pid_t payload_pid = sp_clone_spawn_payload(argv, envp, session, stdout_pipe,
stderr_pipe);
bench_phase_end("payload_fork", "pid1");
close_payload_setup_fds(stdout_pipe, stderr_pipe, session);
close_extra_fds_for_exec(report_fd);
for (;;) {
int status = 0;
sp_init_forward_pending_signal(payload_pid);
pid_t got = waitpid(-1, &status, 0);
if (got < 0) {
if (errno == EINTR)
continue;
if (errno == ECHILD)
return 1;
DBG("init waitpid failed: %s", strerror(errno));
return 1;
}
if (got == payload_pid) {
int payload_status = status;
for (;;) {
got = waitpid(-1, &status, WNOHANG);
if (got > 0)
continue;
if (got < 0 && errno == EINTR)
continue;
break;
}
return sp_wait_status_to_exit_code(payload_status);
}
}
}
static void copy_ifname(char dst[IFNAMSIZ], const char *src) {
int rc = snprintf(dst, IFNAMSIZ, "%s", src);
if (rc < 0 || rc >= IFNAMSIZ) {
errno = ENAMETOOLONG;
die("ifname too long");
}
}
/* ---------- FD passing ---------- */
static void send_fd(int sock, int fd) {
struct msghdr msg;
memset(&msg, 0, sizeof(msg));
char byte = 'X';
struct iovec iov = {&byte, 1};
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
char cbuf[CMSG_SPACE(sizeof(int))];
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
memcpy(CMSG_DATA(cmsg), &fd, sizeof(int));
if (sendmsg(sock, &msg, 0) != 1)
die("sendmsg");
}
static int sp_vnet_tun_profile_values_valid(mode_t mode, int fd_flags,
int status_flags,
unsigned int interface_flags,
const char *actual_name,
const char *expected_name,
int header_size,
int little_endian) {
unsigned int required = IFF_TUN | IFF_NO_PI | IFF_VNET_HDR;
return S_ISCHR(mode) && (fd_flags & FD_CLOEXEC) != 0 &&
(status_flags & O_NONBLOCK) != 0 &&
(interface_flags &
(IFF_TUN | IFF_TAP | IFF_NO_PI | IFF_VNET_HDR | IFF_MULTI_QUEUE)) ==
required &&
strncmp(actual_name, expected_name, IFNAMSIZ) == 0 &&
header_size == SP_VNET_HDR_BYTES && little_endian == 1;
}
static int tun_fd_profile_valid(int fd, const char *name) {
struct stat st;
struct ifreq ifr;
int fd_flags;
int status_flags;
int hdrsz = 0;
int little_endian = 0;
fd_flags = fcntl(fd, F_GETFD);
status_flags = fcntl(fd, F_GETFL);
if (fd_flags < 0 || status_flags < 0 || fstat(fd, &st) < 0)
return 0;
memset(&ifr, 0, sizeof(ifr));
if (ioctl(fd, TUNGETIFF, &ifr) < 0 ||
ioctl(fd, TUNGETVNETHDRSZ, &hdrsz) < 0 ||
ioctl(fd, TUNGETVNETLE, &little_endian) < 0)
return 0;
return sp_vnet_tun_profile_values_valid(
st.st_mode, fd_flags, status_flags, (unsigned int)ifr.ifr_flags,
ifr.ifr_name, name, hdrsz, little_endian);
}
/* musl defines CMSG_NXTHDR as a macro that compares signed with unsigned
* values, which clang reports (-Wsign-compare) at every use; glibc's is a
* function. One wrapper keeps that suppression in one place. */
static struct cmsghdr *sp_cmsg_next(struct msghdr *msg, struct cmsghdr *cmsg) {
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wsign-compare"
#endif
return CMSG_NXTHDR(msg, cmsg);
#ifdef __clang__
#pragma clang diagnostic pop
#endif
}
static int recv_fd_checked(int sock, const char *expected_name) {
struct msghdr msg;
int received[2] = {-1, -1};
size_t received_count = 0;
int invalid = 0;
memset(&msg, 0, sizeof(msg));
char byte;
struct iovec iov = {&byte, 1};
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
char cbuf[CMSG_SPACE(sizeof(received))];
memset(cbuf, 0, sizeof(cbuf));
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
ssize_t n = recvmsg(sock, &msg, MSG_CMSG_CLOEXEC);
if (n < 0)
return -1;
if (n != 1 || byte != 'X' ||
(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) != 0)
invalid = 1;
for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
cmsg = sp_cmsg_next(&msg, cmsg)) {
size_t payload_len;
size_t fd_count;
if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS ||
cmsg->cmsg_len < CMSG_LEN(0)) {
invalid = 1;
continue;
}
payload_len = cmsg->cmsg_len - CMSG_LEN(0);
if (payload_len == 0 || payload_len % sizeof(int) != 0) {
invalid = 1;
continue;
}
fd_count = payload_len / sizeof(int);
if (fd_count > sizeof(received) / sizeof(received[0]) - received_count) {
invalid = 1;
fd_count = sizeof(received) / sizeof(received[0]) - received_count;
}
memcpy(received + received_count, CMSG_DATA(cmsg),
fd_count * sizeof(int));
received_count += fd_count;
}
if (received_count != 1)
invalid = 1;
if (!invalid && !tun_fd_profile_valid(received[0], expected_name))
invalid = 1;
if (invalid) {
for (size_t i = 0; i < received_count; ++i)
if (received[i] >= 0)
close(received[i]);
errno = EPROTO;
return -1;
}
return received[0];
}
/* ---------- TUN and interface helpers for child process ---------- */
static unsigned int sp_vnet_tun_offload_request(void) {
return TUN_F_CSUM | TUN_F_TSO4;
}
/* Opened by PID 1 before pivot_root, while the host /dev/net/tun is still
* reachable: the fd is bound to the opener's (already private) network
* namespace at open time, so no /dev/net projection is needed later. */
static int tun_open(void) {
int fd = open("/dev/net/tun", O_RDWR | O_CLOEXEC | O_NONBLOCK);
if (fd < 0)
die("open /dev/net/tun");
return fd;
}
static int tun_create(int fd, const char *name) {
struct ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
ifr.ifr_flags = IFF_TUN | IFF_NO_PI | IFF_VNET_HDR;
copy_ifname(ifr.ifr_name, name);
if (ioctl(fd, TUNSETIFF, &ifr) < 0)
die("TUNSETIFF");
int hdrsz = SP_VNET_HDR_BYTES;
int little_endian = 1;
unsigned int offloads = sp_vnet_tun_offload_request();
if (ioctl(fd, TUNSETVNETHDRSZ, &hdrsz) < 0)
die("TUNSETVNETHDRSZ");
if (ioctl(fd, TUNSETVNETLE, &little_endian) < 0)
die("TUNSETVNETLE");
if (ioctl(fd, TUNSETOFFLOAD, offloads) < 0)
die("TUNSETOFFLOAD CSUM|TSO4");
if (!tun_fd_profile_valid(fd, name)) {
errno = EPROTO;
die("TUN VNET profile validation");
}
DBG("TUN fd profile validated: name=%s header=%d endian=little "
"cloexec=yes nonblocking=yes",
name, SP_VNET_HDR_BYTES);
return fd;
}
/* PID 1's interface setup shares one socket of each kind instead of opening a
* fresh socket per step. Both are closed once setup finishes. */
struct sp_net_setup {
int inet_fd;
int rtnl_fd;
uint32_t rtnl_seq;
};
static void sp_net_setup_open(struct sp_net_setup *net) {
struct sockaddr_nl address = {.nl_family = AF_NETLINK};
net->inet_fd = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
if (net->inet_fd < 0)
die("socket");
net->rtnl_fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE);
if (net->rtnl_fd < 0 ||
bind(net->rtnl_fd, (struct sockaddr *)&address, sizeof(address)) < 0)
die("rtnetlink socket");
net->rtnl_seq = 0;
}
static void sp_net_setup_close(struct sp_net_setup *net) {
sp_close_owned_fd_nointr(&net->inet_fd);
sp_close_owned_fd_nointr(&net->rtnl_fd);
}
static void if_up(struct sp_net_setup *net, const char *ifname) {
struct ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
copy_ifname(ifr.ifr_name, ifname);
if (ioctl(net->inet_fd, SIOCGIFFLAGS, &ifr) < 0)
die("SIOCGIFFLAGS");
ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
if (ioctl(net->inet_fd, SIOCSIFFLAGS, &ifr) < 0)
die("SIOCSIFFLAGS");
}
/* point-to-point address */
static void if_addr_ptp(struct sp_net_setup *net, const char *ifname,
const char *local, const char *peer) {
struct ifreq ifr = {0};
struct sockaddr_in addr = {.sin_family = AF_INET};
copy_ifname(ifr.ifr_name, ifname);
if (inet_pton(AF_INET, local, &addr.sin_addr) != 1)
die("inet_pton local");
memcpy(&ifr.ifr_addr, &addr, sizeof(addr));
if (ioctl(net->inet_fd, SIOCSIFADDR, &ifr) < 0)
die("SIOCSIFADDR");
if (inet_pton(AF_INET, peer, &addr.sin_addr) != 1)
die("inet_pton peer");
memcpy(&ifr.ifr_dstaddr, &addr, sizeof(addr));
if (ioctl(net->inet_fd, SIOCSIFDSTADDR, &ifr) < 0)
die("SIOCSIFDSTADDR");
}
static int if_index(struct sp_net_setup *net, const char *ifname) {
struct ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
copy_ifname(ifr.ifr_name, ifname);
if (ioctl(net->inet_fd, SIOCGIFINDEX, &ifr) < 0)
die("SIOCGIFINDEX");
return ifr.ifr_ifindex;
}
static int sp_rtnl_add_u32(struct nlmsghdr *nlh, size_t capacity,
unsigned short type, uint32_t value) {
size_t offset = NLMSG_ALIGN(nlh->nlmsg_len);
size_t length = RTA_LENGTH(sizeof(value));
struct rtattr *attr;
if (offset > capacity || length > capacity - offset)
return -1;
attr = (struct rtattr *)((uint8_t *)nlh + offset);
attr->rta_type = type;
attr->rta_len = (unsigned short)length;
memcpy(RTA_DATA(attr), &value, sizeof(value));
nlh->nlmsg_len = (uint32_t)(offset + RTA_ALIGN(length));
return 0;
}
static int sp_rtnl_expect_ack(int fd, uint32_t sequence) {
uint8_t response[4096];
ssize_t length;
do {
length = recv(fd, response, sizeof(response), 0);
} while (length < 0 && errno == EINTR);
if (length <= 0)
return -1;
for (struct nlmsghdr *nlh = (struct nlmsghdr *)response;
NLMSG_OK(nlh, length); nlh = NLMSG_NEXT(nlh, length)) {
struct nlmsgerr *error;
if (nlh->nlmsg_seq != sequence || nlh->nlmsg_type != NLMSG_ERROR ||
nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*error)))
continue;
error = NLMSG_DATA(nlh);
if (error->error == 0)
return 0;
errno = -error->error;
return -1;
}
errno = EPROTO;
return -1;
}
static int sp_link_gso_readback(int fd, int index, uint32_t sequence,
uint32_t expected_size,
uint32_t expected_segs) {
struct {
struct nlmsghdr nlh;
struct ifinfomsg ifi;
} request = {0};
uint8_t response[8192];
ssize_t length;
request.nlh.nlmsg_len = NLMSG_LENGTH(sizeof(request.ifi));
request.nlh.nlmsg_type = RTM_GETLINK;
request.nlh.nlmsg_flags = NLM_F_REQUEST;
request.nlh.nlmsg_seq = sequence;
request.ifi.ifi_family = AF_UNSPEC;
request.ifi.ifi_index = index;
if (send(fd, &request, request.nlh.nlmsg_len, 0) !=
(ssize_t)request.nlh.nlmsg_len)
return -1;
do {
length = recv(fd, response, sizeof(response), 0);
} while (length < 0 && errno == EINTR);
if (length <= 0)
return -1;
for (struct nlmsghdr *nlh = (struct nlmsghdr *)response;
NLMSG_OK(nlh, length); nlh = NLMSG_NEXT(nlh, length)) {
struct ifinfomsg *ifi;
struct rtattr *attr;
int attr_len;
uint32_t size = 0;
uint32_t segs = 0;
if (nlh->nlmsg_seq != sequence || nlh->nlmsg_type != RTM_NEWLINK ||
nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*ifi)))
continue;
ifi = NLMSG_DATA(nlh);
if (ifi->ifi_index != index)
continue;
attr = IFLA_RTA(ifi);
if (IFLA_PAYLOAD(nlh) > INT_MAX) {
errno = EOVERFLOW;
return -1;
}
attr_len = (int)IFLA_PAYLOAD(nlh);
while (attr_len >= (int)sizeof(*attr) &&
attr->rta_len >= sizeof(*attr) && attr->rta_len <= attr_len) {
size_t step = RTA_ALIGN(attr->rta_len);
if (RTA_PAYLOAD(attr) == sizeof(uint32_t)) {
if (attr->rta_type == IFLA_GSO_MAX_SIZE)
memcpy(&size, RTA_DATA(attr), sizeof(size));
else if (attr->rta_type == IFLA_GSO_MAX_SEGS)
memcpy(&segs, RTA_DATA(attr), sizeof(segs));
}
if (step > (size_t)attr_len)
break;
attr_len -= (int)step;
attr = (struct rtattr *)((uint8_t *)attr + step);
}
if (size == expected_size && segs == expected_segs)
return 0;
errno = EPROTO;
return -1;
}
errno = EPROTO;
return -1;
}
/* Send one rtnetlink change and require its acknowledgement: a successful
* send() only means the kernel queued the request, not that it applied it. */
static int sp_rtnl_request_ack(struct sp_net_setup *net, struct nlmsghdr *nlh) {
nlh->nlmsg_flags |= NLM_F_ACK;
nlh->nlmsg_seq = ++net->rtnl_seq;
if (send(net->rtnl_fd, nlh, nlh->nlmsg_len, 0) != (ssize_t)nlh->nlmsg_len)
return -1;
return sp_rtnl_expect_ack(net->rtnl_fd, nlh->nlmsg_seq);
}
static int configure_link_gso(struct sp_net_setup *net, const char *ifname,
int index, uint32_t max_size,
uint32_t max_segs) {
struct {
struct nlmsghdr nlh;
struct ifinfomsg ifi;
uint8_t attributes[64];
} request = {0};
request.nlh.nlmsg_len = NLMSG_LENGTH(sizeof(request.ifi));
request.nlh.nlmsg_type = RTM_NEWLINK;
request.nlh.nlmsg_flags = NLM_F_REQUEST;
request.ifi.ifi_family = AF_UNSPEC;
request.ifi.ifi_index = index;
if (sp_rtnl_add_u32(&request.nlh, sizeof(request), IFLA_GSO_MAX_SIZE,
max_size) < 0 ||
sp_rtnl_add_u32(&request.nlh, sizeof(request), IFLA_GSO_MAX_SEGS,
max_segs) < 0 ||
sp_rtnl_request_ack(net, &request.nlh) < 0 ||
sp_link_gso_readback(net->rtnl_fd, index, ++net->rtnl_seq, max_size,
max_segs) < 0)
return -1;
DBG("TUN GSO limits validated by rtnetlink readback: ifname=%s "
"gso_max_size=%u gso_max_segs=%u",
ifname, max_size, max_segs);
return 0;
}
static void if_up_netlink(struct sp_net_setup *net, int index) {
struct {
struct nlmsghdr nlh;
struct ifinfomsg ifi;
} req = {0};
req.nlh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
req.nlh.nlmsg_type = RTM_NEWLINK;
req.nlh.nlmsg_flags = NLM_F_REQUEST;
req.ifi.ifi_family = AF_UNSPEC;
/* Test hook: name a nonexistent interface so the kernel rejects the
* request and the acknowledgement path sees a real error. */
req.ifi.ifi_index = index;
req.ifi.ifi_flags = IFF_UP | IFF_RUNNING;
req.ifi.ifi_change = IFF_UP | IFF_RUNNING;
if (sp_rtnl_request_ack(net, &req.nlh) < 0)
die("netlink link up");
}
static void add_default_route(struct sp_net_setup *net, int index,
const char *gw) {
struct {
struct nlmsghdr nlh;
struct rtmsg rtm;
uint8_t attributes[64];
} req = {0};
struct in_addr gateway;
if (inet_pton(AF_INET, gw, &gateway) != 1)
die("inet_pton gateway");
req.nlh.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
req.nlh.nlmsg_type = RTM_NEWROUTE;
req.nlh.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE;
req.rtm.rtm_family = AF_INET;
req.rtm.rtm_table = RT_TABLE_MAIN;
req.rtm.rtm_protocol = RTPROT_BOOT;
req.rtm.rtm_scope = RT_SCOPE_UNIVERSE;
req.rtm.rtm_type = RTN_UNICAST;
if (sp_rtnl_add_u32(&req.nlh, sizeof(req), RTA_GATEWAY, gateway.s_addr) < 0 ||
sp_rtnl_add_u32(&req.nlh, sizeof(req), RTA_OIF, (uint32_t)index) < 0 ||
sp_rtnl_request_ack(net, &req.nlh) < 0)
die("netlink default route");
}
/* ----------- TCP/IP helpers ------------------ */
static int dns_is_synthetic_ip(uint32_t ip_be);
/* ---------- Persistent UDP Flow Management ---------- */
static void udp_close_flow(struct udp_flow *f);
static void sp_udp_outbound_key(struct sp_udp_key *key, uint8_t role, uint32_t cip,
uint16_t cport, uint32_t sip,
uint16_t sport) {
memset(key, 0, sizeof(*key));
key->bytes[0] = role;
key->bytes[1] = IPPROTO_UDP;
memcpy(key->bytes + 4, &cip, sizeof(cip));
memcpy(key->bytes + 8, &sip, sizeof(sip));
memcpy(key->bytes + 12, &cport, sizeof(cport));
memcpy(key->bytes + 14, &sport, sizeof(sport));
}
static void sp_udp_publish_key(struct sp_udp_key *key, int rule_index,
uint32_t local_ip, uint16_t local_port,
const struct sockaddr_in *peer) {
memset(key, 0, sizeof(*key));
key->bytes[0] = SP_UDP_ROLE_PUBLISH;
key->bytes[1] = IPPROTO_UDP;
memcpy(key->bytes + 4, &rule_index, sizeof(rule_index));
memcpy(key->bytes + 8, &local_ip, sizeof(local_ip));
memcpy(key->bytes + 12, &local_port, sizeof(local_port));
memcpy(key->bytes + 16, &peer->sin_addr.s_addr, sizeof(peer->sin_addr.s_addr));
memcpy(key->bytes + 20, &peer->sin_port, sizeof(peer->sin_port));
}
static void sp_udp_publish_child_key(struct sp_udp_key *key, uint16_t synthetic_port,
uint16_t child_port) {
memset(key, 0, sizeof(*key));
key->bytes[0] = SP_UDP_ROLE_PUBLISH;
key->bytes[1] = IPPROTO_UDP;
key->bytes[2] = 1;
memcpy(key->bytes + 4, &synthetic_port, sizeof(synthetic_port));
memcpy(key->bytes + 6, &child_port, sizeof(child_port));
memcpy(key->bytes + 8, &(uint32_t){PUBLISH_SYNTHETIC_IP}, sizeof(uint32_t));
memcpy(key->bytes + 12, &(uint32_t){SANDBOX_IP}, sizeof(uint32_t));
}
/* A probe is private to the typed operation; exhaustion/corruption indicates
* an internal invariant failure, not an absent network tuple. */
static enum ht_step sp_udp_map_probe(const struct sp_udp_map *m,
const struct sp_udp_key *key, struct ht_probe *p, uint32_t *slot,
uint32_t *bucket) {
*p = ht_probe_begin(&m->index, ht_siphash13(key->bytes, sizeof(key->bytes), &m->secret));
enum ht_step step;
while ((step = ht_probe_next(&m->index, p, slot, bucket)) == HT_CANDIDATE)
if (memcmp(m->entries[*slot].key.bytes, key->bytes, sizeof(key->bytes)) == 0)
break;
if (step == HT_CORRUPT || step == HT_EXHAUSTED) abort();
return step;
}
static struct sp_udp_entry *sp_udp_table_lookup_entry(const struct sp_udp_key *key) {
struct ht_probe p;
uint32_t slot, bucket;
return sp_udp_map_probe(&g_udp_map, key, &p, &slot, &bucket) == HT_CANDIDATE
? &g_udp_map.entries[slot] : NULL;
}
static int sp_udp_table_insert(const struct sp_udp_key *key, uint32_t generation,
uint16_t index, uint8_t role) {
struct sp_udp_map *m = &g_udp_map;
struct ht_probe p;
uint32_t slot, bucket;
assert(role == SP_UDP_ROLE_DIRECT || role == SP_UDP_ROLE_SOCKS || role == SP_UDP_ROLE_PUBLISH);
if (!generation || (role == SP_UDP_ROLE_PUBLISH ? index >= MAX_PUBLISH_UDP : index >= MAX_UDP))
return -1;
if (sp_udp_map_probe(m, key, &p, &slot, &bucket) == HT_CANDIDATE || !m->free_count)
return -1;
slot = m->free_slots[m->free_count - 1];
m->entries[slot] = (struct sp_udp_entry){.key = *key, .generation = generation,
.value = {.index = index, .role = role}};
if (ht_insert_at(&m->index, &p, bucket, slot) != HT_INSERTED) abort();
--m->free_count;
return 0;
}
static void sp_udp_table_remove(const struct sp_udp_key *key, uint32_t generation) {
struct sp_udp_map *m = &g_udp_map;
struct ht_probe p;
uint32_t slot, bucket;
if (sp_udp_map_probe(m, key, &p, &slot, &bucket) == HT_CANDIDATE &&
m->entries[slot].generation == generation) {
if (ht_erase_at(&m->index, bucket) != slot) abort();
m->free_slots[m->free_count++] = slot;
}
}
static void udp_mark_progress(struct udp_flow *f, uint64_t now_ns) {
(void)sp_udp_runtime_step(&f->core, SP_UDP_PROGRESS, now_ns, 0);
}
static void udp_arm_idle(struct udp_flow *f, uint64_t now_ns) {
(void)sp_udp_runtime_step(&f->core, SP_UDP_READY, now_ns, 0);
}
static uint8_t sp_udp_admission_role(uint8_t role) {
return (uint8_t)(0x80U | role);
}
enum {
SP_UDP_ADMISSION_RULE = 0xc3,
SP_UDP_ADMISSION_GLOBAL = 0xfe,
};
static int sp_udp_admit_key(uint32_t value, uint8_t admission_role) {
struct sp_admission_key key;
struct sp_admission_entry *entry;
sp_resource_admission_source_key(&key, value, admission_role);
entry = sp_resource_admission_acquire(&key);
if (entry == NULL)
return -1;
if (!sp_resource_admission_take(entry, sp_ns_now_ns())) {
sp_resource_admission_release_source(value, admission_role);
return -1;
}
return 0;
}
static int sp_udp_admit_source(uint32_t source_ip, uint8_t role) {
return sp_udp_admit_key(source_ip, sp_udp_admission_role(role));
}
static int sp_udp_admit_global(void) {
return sp_udp_admit_key(0, SP_UDP_ADMISSION_GLOBAL);
}
static int sp_udp_admit_rule(int rule_index) {
return sp_udp_admit_key((uint32_t)rule_index, SP_UDP_ADMISSION_RULE);
}
static void udp_flow_begin(struct udp_flow *f, uint8_t role) {
uint32_t generation = epoll_next_generation(f->generation);
memset(f, 0, sizeof(*f));
f->generation = generation;
f->used = 1;
(void)sp_udp_runtime_step(&f->core, SP_UDP_BEGIN, 0, role);
f->tcp_ctrl = -1;
f->udp_relay = -1;
f->udp_staging = -1;
}
static struct udp_flow *udp_find(uint32_t cip, uint16_t cport, uint32_t sip,
uint16_t sport, uint8_t role) {
struct sp_udp_key key;
struct sp_udp_entry *entry;
struct sp_udp_table_value value;
struct udp_flow *f;
sp_udp_outbound_key(&key, role, cip, cport, sip, sport);
entry = sp_udp_table_lookup_entry(&key);
if (entry == NULL)
return NULL;
value = entry->value;
if (value.role != role || value.index >= (uint16_t)MAX_UDP)
return NULL;
f = &udp_flows[value.index];
if (!f->used || f->generation != entry->generation || f->core.role != role ||
f->cli_ip != cip || f->cli_port != cport || f->srv_ip != sip ||
f->srv_port != sport)
return NULL;
return f;
}
static struct udp_flow *udp_alloc(int socks_role) {
int role_count = 0;
int role_limit = socks_role ? SP_UDP_SOCKS_MAX : SP_UDP_DIRECT_MAX;
for (int i = 0; i < udp_flow_limit; ++i) {
int active = udp_flows[i].used;
if (active &&
udp_flows[i].core.role ==
(socks_role ? SP_UDP_ROLE_SOCKS : SP_UDP_ROLE_DIRECT))
role_count++;
}
if (role_count >= role_limit) {
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
DBG("[parent] UDP %s role cap reached (%d); rejecting new mapping",
socks_role ? "SOCKS" : "direct", role_limit);
return NULL;
}
for (int i = 0; i < udp_flow_limit; i++) {
if (!udp_flows[i].used)
return &udp_flows[i];
}
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
DBG("[parent] UDP flow table full (%d slots); rejecting new mapping",
udp_flow_limit);
return NULL;
}
static uint32_t udp_ctrl_events(const struct udp_flow *f) {
uint32_t events = EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP;
if (f->socks.connect_pending || socks_has_pending_tx(&f->socks))
events |= EPOLLOUT;
return events;
}
static void udp_update_events(struct udp_flow *f) {
uint32_t events = EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP;
if (f->udp_relay >= 0 && f->core.pending_count != 0)
events |= EPOLLOUT;
epoll_set_udp(f, EPOLL_CTL_MOD, FD_UDP_RELAY, events);
}
static uint8_t *udp_pending_data(uint16_t slot) {
if (slot >= SP_UDP_PENDING_ALL)
return NULL;
return g_resource_udp_pending_arena + (size_t)slot * SP_UDP_PAYLOAD_MAX;
}
static void udp_pending_release(uint16_t slot) {
struct sp_udp_pending_slot *pending;
if (slot >= SP_UDP_PENDING_ALL)
return;
pending = &g_udp_pending_slots[slot];
if (!pending->used)
return;
if (g_udp_pending_live_bytes >= pending->length)
g_udp_pending_live_bytes -= pending->length;
else
g_udp_pending_live_bytes = 0;
memset(pending, 0, sizeof(*pending));
}
/* Store one datagram in the shared pending arena and append it to its owner's
* queue. Returns 0 when queued, -1 for an oversized datagram, 1 for a counted
* terminal drop at the global byte or slot limit, and 2 for a counted terminal
* drop at the per-flow limit. The owner is recorded for flush validation. */
static int sp_udp_pending_store(struct sp_udp_core *core, uint32_t generation,
uint16_t owner_index, uint8_t owner_publish,
const uint8_t *quote, const uint8_t *data,
size_t len) {
uint16_t slot = UINT16_MAX;
if (len > SP_UDP_PAYLOAD_MAX)
return -1;
if (core->pending_count >= SP_UDP_PENDING_PER_FLOW) {
sp_counter_inc(&g_resource_counters.udp_queue_slot_exhaustions);
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
return 2;
}
if (g_udp_pending_live_bytes > SP_UDP_PENDING_BYTES - len) {
sp_counter_inc(&g_resource_counters.udp_queue_byte_exhaustions);
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
return 1;
}
for (uint16_t i = 0; i < SP_UDP_PENDING_ALL; ++i) {
if (!g_udp_pending_slots[i].used) {
slot = i;
break;
}
}
if (slot == UINT16_MAX) {
sp_counter_inc(&g_resource_counters.udp_queue_slot_exhaustions);
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
return 1;
}
g_udp_pending_slots[slot].used = 1;
g_udp_pending_slots[slot].generation = generation;
g_udp_pending_slots[slot].length = (uint16_t)len;
g_udp_pending_slots[slot].owner_index = owner_index;
g_udp_pending_slots[slot].owner_role = core->role;
g_udp_pending_slots[slot].owner_publish = owner_publish;
if (quote != NULL) {
memcpy(g_udp_pending_slots[slot].quote, quote,
sizeof(g_udp_pending_slots[slot].quote));
g_udp_pending_slots[slot].has_quote = 1;
}
memcpy(udp_pending_data(slot), data, len);
g_udp_pending_live_bytes += len;
(void)sp_udp_queue(core, slot);
return 0;
}
static int udp_queue_pending(struct udp_flow *f, const uint8_t *data, size_t len,
const uint8_t *canonical_quote) {
int rc = sp_udp_pending_store(&f->core, f->generation,
(uint16_t)(f - udp_flows), 0, canonical_quote,
data, len);
if (rc == 2) {
f->dropped_backpressure++;
DBG("UDP pending queue full; dropping newest datagram (drops=%lu)",
f->dropped_backpressure);
return 1;
}
if (rc == 0)
udp_update_events(f);
return rc;
}
static int udp_socks_setup(struct udp_flow *f, struct socks_config *cfg) {
int tcp_sock = socket(AF_INET, SOCK_STREAM, 0);
int udp_sock = -1;
struct sockaddr_in local = {.sin_family = AF_INET,
.sin_addr.s_addr = htonl(INADDR_ANY),
.sin_port = 0};
socklen_t locallen = sizeof(local);
int rc;
const char *option_reason = "unknown";
if (tcp_sock < 0)
return -1;
udp_sock = socket(AF_INET, SOCK_DGRAM, 0);
if (udp_sock < 0) {
close(tcp_sock);
return -1;
}
if (sp_udp_configure_host_socket(
udp_sock, &f->effective_sndbuf, &f->effective_rcvbuf,
&option_reason) < 0 ||
bind(udp_sock, (struct sockaddr *)&local, sizeof(local)) < 0) {
DBG("SOCKS UDP option/bind setup failed: %s", option_reason);
close(udp_sock);
close(tcp_sock);
return -1;
}
if (getsockname(udp_sock, (struct sockaddr *)&local, &locallen) < 0) {
close(udp_sock);
close(tcp_sock);
return -1;
}
if (set_nonblocking(udp_sock) < 0) {
close(udp_sock);
close(tcp_sock);
return -1;
}
socks_io_reset(&f->socks);
f->socks.is_udp = 1;
f->socks.target_ip = 0;
f->socks.target_port = ntohs(local.sin_port);
rc = start_nonblocking_connect(tcp_sock, &cfg->addr);
if (rc < 0) {
close(udp_sock);
close(tcp_sock);
return -1;
}
f->tcp_ctrl = tcp_sock;
f->udp_relay = -1;
f->udp_staging = udp_sock;
(void)sp_udp_runtime_step(&f->core, SP_UDP_NEGOTIATE, sp_ns_now_ns(), 0);
f->socks.connect_pending = (rc > 0);
f->socks.state = f->socks.connect_pending ? SOCKS_IO_CONNECTING
: SOCKS_IO_METHOD;
if (!f->socks.connect_pending && socks_begin_handshake(&f->socks, cfg) < 0) {
close(udp_sock);
close(tcp_sock);
f->tcp_ctrl = -1;
f->udp_staging = -1;
return -1;
}
return 0;
}
static int udp_direct_setup(struct udp_flow *f, uint32_t dst_ip,
uint16_t dst_port) {
int udp_sock = socket(AF_INET, SOCK_DGRAM, 0);
const char *option_reason = "unknown";
if (udp_sock < 0)
return -1;
struct sockaddr_in dst = {
.sin_family = AF_INET,
.sin_port = htons(dst_port),
.sin_addr.s_addr = dst_ip,
};
if (sp_udp_configure_host_socket(
udp_sock, &f->effective_sndbuf, &f->effective_rcvbuf,
&option_reason) < 0 ||
connect(udp_sock, (struct sockaddr *)&dst, sizeof(dst)) < 0) {
DBG("direct UDP option/connect setup failed: %s", option_reason);
close(udp_sock);
return -1;
}
if (set_nonblocking(udp_sock) < 0) {
close(udp_sock);
return -1;
}
f->tcp_ctrl = -1;
f->udp_relay = udp_sock;
f->relay_addr = dst;
f->connected_remote = dst;
udp_arm_idle(f, sp_ns_now_ns());
return 0;
}
enum sp_udp_send_result {
SP_UDP_SEND_FATAL = -1,
SP_UDP_SEND_DONE = 0,
SP_UDP_SEND_RETAIN = 1,
SP_UDP_SEND_EMSGSIZE = 2,
SP_UDP_SEND_REFUSED = 3,
SP_UDP_SEND_DROPPED = 4,
};
/* A positive result retains ownership only for RETAIN. MTU/refusal results
* are terminal for the datagram and let the caller synthesize the narrowly
* permitted ICMP result from its canonical quote. */
static int udp_socks_mapping(const struct udp_flow *f,
struct sp_dns_name_view *mapping) {
/* A synthetic destination must stay tied to the mapping acquired at flow
* admission. Peeking an unpinned slot here could silently encode a new
* name after expiry/reuse; numeric destinations simply use their address. */
if (f == NULL || mapping == NULL || f->dns_mapping_ref.generation == 0)
return 0;
if (sp_dns_core_mapping_name(&g_dns_core, f->dns_mapping_ref, mapping) < 0)
return 0;
return mapping->synthetic_ip == f->srv_ip;
}
static int udp_socks_encode_packet(const struct sp_net_policy_snapshot *policy,
const struct udp_flow *f, const uint8_t *payload, size_t payload_len,
uint8_t *packet, size_t capacity, size_t *packet_len) {
struct sp_dns_name_view mapping;
int mapped = policy->socks_remote_dns && udp_socks_mapping(f, &mapping);
if (!mapped && policy->socks_remote_dns && dns_is_synthetic_ip(f->srv_ip))
return -1;
return sp_socks_udp_encode((struct sp_socks_endpoint){f->srv_ip, f->srv_port},
mapped ? mapping.name : NULL, mapped ? mapping.name_len : 0,
payload, payload_len, packet, capacity, packet_len);
}
static int udp_send_socks_now(struct udp_flow *f, const uint8_t *payload,
size_t payload_len) {
uint8_t pkt[65536];
struct in_addr dst_addr = {.s_addr = f->srv_ip};
size_t packet_len;
unsigned int interrupted = 0;
if (udp_socks_encode_packet(&g_net_policy, f, payload, payload_len, pkt,
sizeof(pkt), &packet_len) < 0)
return -1;
if (verbose) {
/* Separate buffers: inet_ntoa() returns one shared static buffer. */
char relay_text[INET_ADDRSTRLEN], dst_text[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &f->relay_addr.sin_addr, relay_text, sizeof(relay_text));
inet_ntop(AF_INET, &dst_addr, dst_text, sizeof(dst_text));
DBG("UDP via SOCKS relay %s:%d -> %s:%d len=%zu", relay_text,
ntohs(f->relay_addr.sin_port), dst_text, f->srv_port, payload_len);
}
for (;;) {
ssize_t n = sendto(f->udp_relay, pkt, packet_len, 0,
(struct sockaddr *)&f->relay_addr,
sizeof(f->relay_addr));
sp_resource_count_host_io(1, 1, n);
if (n == (ssize_t)packet_len) {
udp_mark_progress(f, sp_ns_now_ns());
return 0;
}
if (n >= 0) {
DBG("SOCKS UDP short send: %zd/%zu, closing flow", n, packet_len);
return -1;
}
if (errno == EINTR && interrupted++ < 4)
continue;
if (errno == EINTR)
return SP_UDP_SEND_RETAIN;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return SP_UDP_SEND_RETAIN;
if (errno == EMSGSIZE)
return SP_UDP_SEND_EMSGSIZE;
if (errno == ECONNREFUSED)
return SP_UDP_SEND_REFUSED;
if (errno == ENOBUFS || errno == ENOMEM) {
f->dropped_backpressure++;
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
DBG("SOCKS UDP send drop due to %s (drops=%lu)", strerror(errno),
f->dropped_backpressure);
return SP_UDP_SEND_DROPPED;
}
return -1;
}
}
static int udp_socks_decode_payload(const struct udp_flow *f,
const uint8_t *packet, size_t packet_len, const uint8_t **payload,
size_t *payload_len) {
struct sp_dns_name_view mapping;
int mapped = packet_len >= 4 && packet[3] == 3 && udp_socks_mapping(f, &mapping);
return sp_socks_udp_decode((struct sp_socks_endpoint){f->srv_ip, f->srv_port},
mapped ? mapping.name : NULL, mapped ? mapping.name_len : 0,
packet, packet_len, payload, payload_len);
}
static int udp_source_matches(const struct udp_flow *f,
const struct sockaddr_in *source,
socklen_t source_len) {
return f != NULL && source != NULL && source_len == sizeof(*source) &&
source->sin_family == AF_INET &&
source->sin_addr.s_addr == f->connected_remote.sin_addr.s_addr &&
source->sin_port == f->connected_remote.sin_port;
}
enum sp_udp_mmsg_receive_result {
SP_UDP_MMSG_RECEIVE_ERROR = -1,
};
static int sp_udp_emit_error(int tunfd, const uint8_t *quote,
const struct udp_flow *f,
enum sp_udp_send_result result);
struct sp_packet_value;
static int sp_udp_dispatch_values(const uint8_t *packet, size_t packet_len,
const struct sp_packet_value *parsed,
uint32_t *src_ip, uint32_t *dst_ip,
uint16_t *src_port, uint16_t *dst_port,
const uint8_t **payload,
size_t *payload_len);
static void sp_udp_emit_tun(int tunfd, uint32_t src_ip, uint32_t dst_ip,
uint16_t src_port, uint16_t dst_port,
const uint8_t *payload, size_t payload_len,
const char *context);
static unsigned int sp_udp_mmsg_prepare(unsigned int requested) {
unsigned int count =
requested > SP_UDP_MMSG_MAX ? SP_UDP_MMSG_MAX : requested;
for (unsigned int i = 0; i < count; ++i) {
memset(&g_udp_mmsg_messages[i], 0, sizeof(g_udp_mmsg_messages[i]));
memset(&g_udp_mmsg_iov[i], 0, sizeof(g_udp_mmsg_iov[i]));
memset(&g_udp_mmsg_sources[i], 0, sizeof(g_udp_mmsg_sources[i]));
memset(g_udp_mmsg_control[i], 0, sizeof(g_udp_mmsg_control[i]));
memset(g_udp_mmsg_payload[i], 0, sizeof(g_udp_mmsg_payload[i]));
g_udp_mmsg_iov[i].iov_base = g_udp_mmsg_payload[i];
g_udp_mmsg_iov[i].iov_len = sizeof(g_udp_mmsg_payload[i]);
g_udp_mmsg_messages[i].msg_hdr.msg_name = &g_udp_mmsg_sources[i];
g_udp_mmsg_messages[i].msg_hdr.msg_namelen =
sizeof(g_udp_mmsg_sources[i]);
g_udp_mmsg_messages[i].msg_hdr.msg_iov = &g_udp_mmsg_iov[i];
g_udp_mmsg_messages[i].msg_hdr.msg_iovlen = 1;
g_udp_mmsg_messages[i].msg_hdr.msg_control = g_udp_mmsg_control[i];
g_udp_mmsg_messages[i].msg_hdr.msg_controllen =
sizeof(g_udp_mmsg_control[i]);
}
return count;
}
/* Submit one fd and no timeout. EINTR has no ownership effect and is retried
* only while both the fixed retry count and callback-time budget permit. */
static int sp_udp_recvmmsg_one_fd(int fd, unsigned int requested,
uint64_t started_ns) {
unsigned int count = sp_udp_mmsg_prepare(requested);
unsigned int interrupted = 0;
if (count == 0)
return 0;
for (;;) {
int result;
sp_counter_inc(&g_resource_counters.udp_recvmmsg_calls);
sp_counter_inc(&g_resource_counters.host_datagram_read_calls);
result = g_sp_recvmmsg(fd, g_udp_mmsg_messages, count,
MSG_DONTWAIT | MSG_CMSG_CLOEXEC, NULL);
if (result >= 0) {
uint64_t bytes = 0;
if (result == 0)
sp_counter_inc(&g_resource_counters.udp_recvmmsg_zero_results);
else
sp_counter_add(&g_resource_counters.udp_recvmmsg_datagrams,
(uint64_t)(unsigned int)result);
for (int i = 0; i < result; ++i)
bytes += g_udp_mmsg_messages[i].msg_len;
sp_counter_add(&g_resource_counters.host_datagram_read_bytes, bytes);
return result;
}
if (errno != EINTR)
return SP_UDP_MMSG_RECEIVE_ERROR;
interrupted++;
if (interrupted >= 4 ||
(started_ns != 0 && sp_ns_now_ns() >= started_ns &&
sp_ns_now_ns() - started_ns >= (uint64_t)SP_CALLBACK_BUDGET_NS)) {
sp_counter_inc(&g_resource_counters.udp_recvmmsg_eintr_yields);
errno = EINTR;
return SP_UDP_MMSG_RECEIVE_ERROR;
}
sp_udp_mmsg_prepare(count);
}
}
static int sp_udp_mmsg_slot_basic_valid(unsigned int index) {
const struct mmsghdr *slot;
const struct msghdr *message;
if (index >= SP_UDP_MMSG_MAX)
return 0;
slot = &g_udp_mmsg_messages[index];
message = &slot->msg_hdr;
return slot->msg_len <= sizeof(g_udp_mmsg_payload[index]) &&
(message->msg_flags & (MSG_TRUNC | MSG_CTRUNC)) == 0 &&
message->msg_namelen == sizeof(g_udp_mmsg_sources[index]) &&
g_udp_mmsg_sources[index].sin_family == AF_INET &&
message->msg_controllen <= sizeof(g_udp_mmsg_control[index]);
}
enum sp_udp_mmsg_send_result {
SP_UDP_MMSG_SEND_ERROR = -1,
};
static unsigned int sp_udp_mmsg_tx_reset(unsigned int requested) {
unsigned int count =
requested > SP_UDP_MMSG_MAX ? SP_UDP_MMSG_MAX : requested;
for (unsigned int i = 0; i < count; ++i) {
memset(&g_udp_mmsg_tx_messages[i], 0,
sizeof(g_udp_mmsg_tx_messages[i]));
memset(&g_udp_mmsg_tx_iov[i], 0, sizeof(g_udp_mmsg_tx_iov[i]));
memset(&g_udp_mmsg_tx_names[i], 0, sizeof(g_udp_mmsg_tx_names[i]));
memset(g_udp_mmsg_tx_control[i], 0,
sizeof(g_udp_mmsg_tx_control[i]));
memset(g_udp_mmsg_tx_payload[i], 0, sizeof(g_udp_mmsg_tx_payload[i]));
g_udp_mmsg_tx_messages[i].msg_hdr.msg_iov = &g_udp_mmsg_tx_iov[i];
g_udp_mmsg_tx_messages[i].msg_hdr.msg_iovlen = 1;
}
return count;
}
static int sp_udp_sendmmsg_one_fd(int fd, unsigned int count,
uint64_t started_ns) {
unsigned int interrupted = 0;
if (count == 0 || count > SP_UDP_MMSG_MAX) {
errno = EINVAL;
return count == 0 ? 0 : SP_UDP_MMSG_SEND_ERROR;
}
for (;;) {
int result;
for (unsigned int i = 0; i < count; ++i)
g_udp_mmsg_tx_messages[i].msg_len = 0;
result = g_sp_sendmmsg(fd, g_udp_mmsg_tx_messages, count,
MSG_DONTWAIT | MSG_NOSIGNAL);
if (result >= 0) {
uint64_t bytes = 0;
if (result == 0)
sp_counter_inc(&g_resource_counters.udp_sendmmsg_zero_results);
else
sp_counter_add(&g_resource_counters.udp_sendmmsg_datagrams,
(uint64_t)(unsigned int)result);
if (result > 0 && (unsigned int)result < count)
sp_counter_inc(&g_resource_counters.udp_sendmmsg_partial_prefixes);
for (int i = 0; i < result; ++i)
bytes += g_udp_mmsg_tx_messages[i].msg_len;
sp_counter_add(&g_resource_counters.host_datagram_write_bytes, bytes);
return result;
}
if (errno != EINTR)
return SP_UDP_MMSG_SEND_ERROR;
interrupted++;
if (interrupted >= 4 ||
(started_ns != 0 && sp_ns_now_ns() >= started_ns &&
sp_ns_now_ns() - started_ns >= (uint64_t)SP_CALLBACK_BUDGET_NS)) {
sp_counter_inc(&g_resource_counters.udp_sendmmsg_eintr_yields);
errno = EINTR;
return SP_UDP_MMSG_SEND_ERROR;
}
}
}
/* UDP I/O uses only recvmmsg/sendmmsg (Linux >= 3.0), and our own seccomp
* policies allow them. An outer runtime policy can still deny them; detect
* that before any flow depends on UDP. With an invalid fd, an available
* syscall fails with EBADF. The probe passes the runtime vector length and
* flags so a policy that pins those arguments (ours does) still accepts it. */
static int sp_udp_mmsg_available(void) {
uint8_t byte = 0;
struct iovec iov = {.iov_base = &byte, .iov_len = 0};
struct mmsghdr message = {.msg_hdr = {.msg_iov = &iov, .msg_iovlen = 1}};
if (g_sp_recvmmsg(-1, &message, 1, MSG_DONTWAIT | MSG_CMSG_CLOEXEC, NULL) >=
0 ||
errno != EBADF)
return -1;
if (g_sp_sendmmsg(-1, &message, 1, MSG_DONTWAIT | MSG_NOSIGNAL) >= 0 ||
errno != EBADF)
return -1;
return 0;
}
static int udp_receive_mmsg(int tunfd, struct udp_flow *f) {
uint64_t started_ns = sp_ns_now_ns();
int received =
sp_udp_recvmmsg_one_fd(f->udp_relay, SP_UDP_MMSG_MAX, started_ns);
if (received < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
return 0;
if (errno == ECONNREFUSED && f->has_last_quote)
(void)sp_udp_emit_error(tunfd, f->last_quote, f, SP_UDP_SEND_REFUSED);
return -1;
}
for (int i = 0; i < received; ++i) {
const uint8_t *payload = g_udp_mmsg_payload[i];
size_t payload_len = g_udp_mmsg_messages[i].msg_len;
if (!sp_udp_mmsg_slot_basic_valid((unsigned int)i) ||
g_udp_mmsg_messages[i].msg_hdr.msg_controllen != 0) {
continue;
}
if (!udp_source_matches(f, &g_udp_mmsg_sources[i],
g_udp_mmsg_messages[i].msg_hdr.msg_namelen)) {
continue;
}
if (f->core.role == SP_UDP_ROLE_SOCKS) {
if (udp_socks_decode_payload(f, payload, payload_len, &payload,
&payload_len) < 0) {
continue;
}
}
if (payload_len > SP_UDP_PAYLOAD_MAX) {
continue;
}
sp_udp_emit_tun(tunfd, ntohl(f->srv_ip), ntohl(f->cli_ip), f->srv_port,
f->cli_port, payload, payload_len, "UDP inject");
udp_mark_progress(f, sp_ns_now_ns());
}
return 0;
}
static uint16_t udp_effective_path_mtu(const struct udp_flow *f,
uint16_t rejected_ip_len) {
int mtu = 0;
socklen_t length = sizeof(mtu);
if (f == NULL || !f->used || f->core.state != SP_UDP_MAPPING_READY ||
f->udp_relay < 0 ||
getsockopt(f->udp_relay, IPPROTO_IP, IP_MTU, &mtu, &length) < 0 ||
length != sizeof(mtu) || mtu < 68 || mtu > 1499)
return 0;
if (f->core.role == SP_UDP_ROLE_SOCKS) {
struct sp_dns_name_view mapping;
int have_mapping = udp_socks_mapping(f, &mapping);
if (dns_is_synthetic_ip(f->srv_ip) && !have_mapping)
return 0;
size_t header_len = have_mapping ? 7U + mapping.name_len : 10U;
if ((size_t)mtu <= header_len)
return 0;
mtu -= (int)header_len;
}
if (mtu < 68 || mtu >= rejected_ip_len)
return 0;
return (uint16_t)mtu;
}
static int udp_send_direct_now(struct udp_flow *f, const uint8_t *payload,
size_t payload_len) {
unsigned int interrupted = 0;
for (;;) {
ssize_t n = send(f->udp_relay, payload, payload_len, MSG_NOSIGNAL);
sp_resource_count_host_io(1, 1, n);
if (n == (ssize_t)payload_len) {
udp_mark_progress(f, sp_ns_now_ns());
return 0;
}
if (n >= 0) {
DBG("UDP short send: %zd/%zu, closing flow", n, payload_len);
return -1;
}
if (errno == EINTR && interrupted++ < 4)
continue;
if (errno == EINTR)
return SP_UDP_SEND_RETAIN;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return SP_UDP_SEND_RETAIN;
if (errno == EMSGSIZE)
return SP_UDP_SEND_EMSGSIZE;
if (errno == ECONNREFUSED)
return SP_UDP_SEND_REFUSED;
if (errno == ENOBUFS || errno == ENOMEM) {
f->dropped_backpressure++;
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
DBG("UDP send drop due to %s (drops=%lu)", strerror(errno),
f->dropped_backpressure);
return SP_UDP_SEND_DROPPED;
}
return -1;
}
}
static int sp_udp_emit_error(int tunfd, const uint8_t *quote,
const struct udp_flow *f,
enum sp_udp_send_result result);
static int udp_open_relay_socket(struct udp_flow *f,
const struct sockaddr_in *relay_addr) {
int udp_sock = f->udp_staging;
if (udp_sock < 0)
return -1;
f->udp_relay = udp_sock;
f->udp_staging = -1;
f->relay_addr = *relay_addr;
f->connected_remote = *relay_addr;
udp_arm_idle(f, sp_ns_now_ns());
epoll_set_udp(f, EPOLL_CTL_ADD, FD_UDP_RELAY, EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP);
udp_update_events(f);
return 0;
}
static int udp_flush_pending(int tunfd, struct udp_flow *f) {
uint16_t submitted_slots[SP_UDP_MMSG_MAX];
unsigned int count;
int result;
int invariant_failed = 0;
if (f->udp_relay < 0 || f->core.pending_count == 0)
return 0;
count = sp_udp_mmsg_tx_reset(f->core.pending_count);
for (unsigned int i = 0; i < count; ++i) {
size_t queue_index =
(f->core.pending_head + i) % SP_UDP_PENDING_PER_FLOW;
uint16_t slot = f->core.pending_slots[queue_index];
struct sp_udp_pending_slot *pending;
uint8_t *payload;
size_t wire_len;
if (slot >= SP_UDP_PENDING_ALL)
return -1;
pending = &g_udp_pending_slots[slot];
if (!pending->used || pending->generation != f->generation ||
pending->owner_index != (uint16_t)(f - udp_flows) ||
pending->owner_role != f->core.role || pending->owner_publish)
return -1;
payload = udp_pending_data(slot);
submitted_slots[i] = slot;
if (f->core.role == SP_UDP_ROLE_SOCKS) {
if (udp_socks_encode_packet(&g_net_policy, f, payload, pending->length,
g_udp_mmsg_tx_payload[i],
sizeof(g_udp_mmsg_tx_payload[i]),
&wire_len) < 0)
return -1;
g_udp_mmsg_tx_iov[i].iov_base = g_udp_mmsg_tx_payload[i];
g_udp_mmsg_tx_iov[i].iov_len = wire_len;
g_udp_mmsg_tx_names[i] = f->relay_addr;
g_udp_mmsg_tx_messages[i].msg_hdr.msg_name =
&g_udp_mmsg_tx_names[i];
g_udp_mmsg_tx_messages[i].msg_hdr.msg_namelen =
sizeof(g_udp_mmsg_tx_names[i]);
} else {
g_udp_mmsg_tx_iov[i].iov_base = payload;
g_udp_mmsg_tx_iov[i].iov_len = pending->length;
}
}
result = sp_udp_sendmmsg_one_fd(f->udp_relay, count, sp_ns_now_ns());
if (result == 0) {
errno = EIO;
return -1;
}
if (result < 0) {
uint16_t slot = submitted_slots[0];
struct sp_udp_pending_slot *pending = &g_udp_pending_slots[slot];
int send_result;
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
udp_update_events(f);
return 0;
}
if (errno == EMSGSIZE)
send_result = SP_UDP_SEND_EMSGSIZE;
else if (errno == ECONNREFUSED)
send_result = SP_UDP_SEND_REFUSED;
else if (errno == ENOBUFS || errno == ENOMEM)
send_result = SP_UDP_SEND_DROPPED;
else
return -1;
if ((send_result == SP_UDP_SEND_EMSGSIZE ||
send_result == SP_UDP_SEND_REFUSED) &&
pending->has_quote)
(void)sp_udp_emit_error(tunfd, pending->quote, f,
(enum sp_udp_send_result)send_result);
if (send_result == SP_UDP_SEND_DROPPED)
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
udp_pending_release(slot);
sp_udp_pop(&f->core);
udp_update_events(f);
return 0;
}
for (int i = 0; i < result; ++i) {
uint16_t slot = submitted_slots[i];
if (g_udp_mmsg_tx_messages[i].msg_len !=
g_udp_mmsg_tx_iov[i].iov_len)
invariant_failed = 1;
if (f->core.pending_count == 0 ||
f->core.pending_slots[f->core.pending_head] != slot)
return -1;
udp_pending_release(slot);
sp_udp_pop(&f->core);
udp_mark_progress(f, sp_ns_now_ns());
}
udp_update_events(f);
return invariant_failed ? -1 : 0;
}
static uint16_t dns_get_u16(const uint8_t *p) {
return (uint16_t)(((uint16_t)p[0] << 8) | p[1]);
}
static void dns_put_u16(uint8_t *p, uint16_t v) {
p[0] = (uint8_t)(v >> 8);
p[1] = (uint8_t)(v & 0xff);
}
static void dns_put_u32(uint8_t *p, uint32_t v) {
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)((v >> 16) & 0xff);
p[2] = (uint8_t)((v >> 8) & 0xff);
p[3] = (uint8_t)(v & 0xff);
}
SP_TEST_API static int sp_dns_decode_query(const uint8_t *query, size_t len,
struct sp_dns_query *out) {
struct sp_dns_query candidate;
size_t off = 12;
size_t name_len = 0;
size_t labels = 0;
size_t question_end;
if (query == NULL || out == NULL || len < 12)
return -1;
if (dns_get_u16(query + 4) != 1 || (dns_get_u16(query + 2) & 0x8000U) != 0)
return -1;
memset(&candidate, 0, sizeof(candidate));
candidate.identifier = dns_get_u16(query);
candidate.request_flags = dns_get_u16(query + 2);
for (;;) {
uint8_t label_len;
if (off >= len)
return -1;
label_len = query[off++];
if (label_len == 0)
break;
if ((label_len & 0xc0U) != 0 || label_len > 63 ||
(size_t)label_len > len - off)
return -1;
if (labels != 0) {
if (name_len + 1 >= sizeof(candidate.name))
return -1;
candidate.name[name_len++] = '.';
}
if ((size_t)label_len > sizeof(candidate.name) - 1U - name_len)
return -1;
for (uint8_t i = 0; i < label_len; ++i) {
unsigned char ch = query[off + i];
if (ch <= 0x20 || ch >= 0x7f)
return -1;
if (ch >= 'A' && ch <= 'Z')
ch = (unsigned char)(ch - 'A' + 'a');
candidate.name[name_len++] = (char)ch;
}
off += label_len;
++labels;
}
if (name_len == 0 || off > len - 4U)
return -1;
question_end = off + 4U;
if (question_end - 12U > sizeof(candidate.question) ||
question_end - 12U > SP_DNS_QUESTION_MAX)
return -1;
candidate.name_len = (uint16_t)name_len;
candidate.qtype = dns_get_u16(query + off);
candidate.qclass = dns_get_u16(query + off + 2);
candidate.question_len = (uint16_t)(question_end - 12U);
memcpy(candidate.question, query + 12, candidate.question_len);
*out = candidate;
return 0;
}
static uint64_t sp_dns_core_ttl(const struct sp_dns_core *core) {
return core != NULL && core->ttl_ns != 0 ? core->ttl_ns : SP_DNS_TTL_NS;
}
static uint64_t sp_dns_deadline_ns(uint64_t now_ns, uint64_t ttl_ns) {
return UINT64_MAX - now_ns < ttl_ns ? UINT64_MAX : now_ns + ttl_ns;
}
static struct sp_dns_ref sp_dns_ref_invalid(void) {
return (struct sp_dns_ref){.slot = SP_DNS_SLOT_INVALID, .generation = 0};
}
static int sp_dns_mapping_shape_valid(const struct dns_mapping *mapping) {
return mapping != NULL && mapping->used && mapping->generation != 0 &&
mapping->synthetic_ip != 0 && mapping->name_len != 0 &&
mapping->name_len <= SP_DNS_NAME_MAX &&
mapping->name[mapping->name_len] == '\0';
}
static int sp_dns_lookup_key_valid(const struct sp_dns_lookup_key *key) {
switch (key->kind) {
case SP_DNS_LOOKUP_EXPIRE:
return 1;
case SP_DNS_LOOKUP_NAME:
return key->name != NULL && key->name_len != 0 &&
key->name_len <= SP_DNS_NAME_MAX &&
key->name[key->name_len] == '\0';
case SP_DNS_LOOKUP_IP:
return key->synthetic_ip != 0;
}
return 0;
}
/* Perform the table's only ordinary lookup/selection walk. Expiry is folded
* into the walk so a query does not scan once to expire, again to find its
* key, and again to select a replacement. The walk is bounded by the fixed
* table size and returns indices only; callers decide whether to refresh,
* acquire or replace after validation and output-credit checks. */
static int sp_dns_core_lookup(struct sp_dns_core *core,
const struct sp_dns_lookup_key *key,
uint64_t now_ns,
struct sp_dns_lookup_result *out) {
uint64_t oldest_ns = UINT64_MAX;
if (!sp_dns_lookup_key_valid(key))
return -1;
*out = (struct sp_dns_lookup_result){
.match_slot = SP_DNS_SLOT_INVALID,
.free_slot = SP_DNS_SLOT_INVALID,
.oldest_unpinned_slot = SP_DNS_SLOT_INVALID,
};
for (uint16_t i = 0; i < MAX_DNS_MAPPINGS; ++i) {
struct dns_mapping *mapping = &core->mappings[i];
if (!mapping->used) {
if (out->free_slot == SP_DNS_SLOT_INVALID)
out->free_slot = i;
continue;
}
if (!sp_dns_mapping_shape_valid(mapping))
return -1;
if (mapping->active_refs == 0 && mapping->expires_ns != 0 &&
now_ns >= mapping->expires_ns) {
memset(mapping, 0, sizeof(*mapping));
if (out->free_slot == SP_DNS_SLOT_INVALID)
out->free_slot = i;
continue;
}
if (key->kind == SP_DNS_LOOKUP_NAME &&
mapping->name_len == key->name_len &&
memcmp(mapping->name, key->name, key->name_len) == 0) {
if (out->match_slot == SP_DNS_SLOT_INVALID)
out->match_slot = i;
} else if (key->kind == SP_DNS_LOOKUP_IP &&
mapping->synthetic_ip == key->synthetic_ip) {
if (out->match_slot == SP_DNS_SLOT_INVALID)
out->match_slot = i;
}
if (key->kind == SP_DNS_LOOKUP_NAME && mapping->active_refs == 0 &&
(out->oldest_unpinned_slot == SP_DNS_SLOT_INVALID ||
mapping->last_used_ns < oldest_ns)) {
out->oldest_unpinned_slot = i;
oldest_ns = mapping->last_used_ns;
}
}
return 0;
}
static int sp_dns_core_ip_in_use(const struct sp_dns_core *core,
uint32_t synthetic_ip) {
if (synthetic_ip == 0)
return 0;
for (uint16_t i = 0; i < MAX_DNS_MAPPINGS; ++i)
if (core->mappings[i].used &&
core->mappings[i].synthetic_ip == synthetic_ip)
return 1;
return 0;
}
static uint32_t sp_dns_core_next_ip(struct sp_dns_core *core) {
uint32_t cursor;
cursor = core->next_host;
if (cursor == 0 || cursor > 0xfffeU)
cursor = 1;
for (uint32_t attempt = 0; attempt < 0xfffeU; ++attempt) {
uint32_t host = cursor;
uint32_t ip;
cursor = host == 0xfffeU ? 1U : host + 1U;
ip = htonl(DNS_SYNTHETIC_BASE | host);
if (!sp_dns_core_ip_in_use(core, ip)) {
core->next_host = cursor;
return ip;
}
}
core->next_host = cursor;
return 0;
}
static uint32_t sp_dns_core_next_generation(struct sp_dns_core *core) {
uint32_t generation;
generation = core->next_generation;
if (generation == 0)
generation = 1;
core->next_generation = generation == UINT32_MAX ? 1U : generation + 1U;
return generation;
}
static int sp_dns_core_get_or_create(
struct sp_dns_core *core, const char *name, uint16_t name_len,
uint64_t now_ns, uint32_t *synthetic_ip) {
struct sp_dns_lookup_key key;
struct sp_dns_lookup_result lookup;
struct dns_mapping *mapping;
uint16_t slot;
uint32_t candidate_ip;
uint32_t next_generation;
*synthetic_ip = 0;
key = (struct sp_dns_lookup_key){
.kind = SP_DNS_LOOKUP_NAME, .name = name, .name_len = name_len};
if (sp_dns_core_lookup(core, &key, now_ns, &lookup) < 0)
return -1;
if (lookup.match_slot != SP_DNS_SLOT_INVALID) {
mapping = &core->mappings[lookup.match_slot];
mapping->last_used_ns = now_ns;
mapping->expires_ns = sp_dns_deadline_ns(now_ns, sp_dns_core_ttl(core));
*synthetic_ip = mapping->synthetic_ip;
return 0;
}
slot = lookup.free_slot != SP_DNS_SLOT_INVALID
? lookup.free_slot
: lookup.oldest_unpinned_slot;
if (slot == SP_DNS_SLOT_INVALID)
return -1;
mapping = &core->mappings[slot];
candidate_ip = sp_dns_core_next_ip(core);
if (candidate_ip == 0)
return -1;
next_generation = sp_dns_core_next_generation(core);
memset(mapping, 0, sizeof(*mapping));
mapping->used = 1;
mapping->generation = next_generation;
mapping->synthetic_ip = candidate_ip;
mapping->last_used_ns = now_ns;
mapping->expires_ns = sp_dns_deadline_ns(now_ns, sp_dns_core_ttl(core));
mapping->name_len = name_len;
memcpy(mapping->name, name, name_len);
mapping->name[name_len] = '\0';
*synthetic_ip = candidate_ip;
return 0;
}
static const struct dns_mapping *sp_dns_core_ref_get(
const struct sp_dns_core *core, struct sp_dns_ref ref) {
const struct dns_mapping *mapping;
if (ref.slot == SP_DNS_SLOT_INVALID || ref.slot >= MAX_DNS_MAPPINGS || ref.generation == 0)
return NULL;
mapping = &core->mappings[ref.slot];
return sp_dns_mapping_shape_valid(mapping) &&
mapping->generation == ref.generation
? mapping
: NULL;
}
static struct dns_mapping *sp_dns_core_ref_get_mut(struct sp_dns_core *core,
struct sp_dns_ref ref) {
return (struct dns_mapping *)sp_dns_core_ref_get(core, ref);
}
SP_TEST_API static int sp_dns_core_mapping_name(
const struct sp_dns_core *core, struct sp_dns_ref ref,
struct sp_dns_name_view *out) {
const struct dns_mapping *mapping;
*out = (struct sp_dns_name_view){0};
mapping = sp_dns_core_ref_get(core, ref);
if (mapping == NULL || mapping->active_refs == 0)
return -1;
out->synthetic_ip = mapping->synthetic_ip;
out->name_len = mapping->name_len;
out->name = mapping->name;
return 0;
}
SP_TEST_API static int sp_dns_core_acquire_ip(
struct sp_dns_core *core, uint32_t synthetic_ip, uint64_t now_ns,
struct sp_dns_ref *out) {
struct sp_dns_lookup_key key;
struct sp_dns_lookup_result lookup;
struct dns_mapping *mapping;
if (synthetic_ip == 0)
return -1;
*out = sp_dns_ref_invalid();
key = (struct sp_dns_lookup_key){
.kind = SP_DNS_LOOKUP_IP, .synthetic_ip = synthetic_ip};
if (sp_dns_core_lookup(core, &key, now_ns, &lookup) < 0 ||
lookup.match_slot == SP_DNS_SLOT_INVALID)
return -1;
mapping = &core->mappings[lookup.match_slot];
if (mapping->active_refs == UINT32_MAX)
return -1;
mapping->active_refs++;
mapping->last_used_ns = now_ns;
mapping->expires_ns = sp_dns_deadline_ns(now_ns, sp_dns_core_ttl(core));
out->slot = lookup.match_slot;
out->generation = mapping->generation;
return 0;
}
SP_TEST_API static int sp_dns_core_release(
struct sp_dns_core *core, struct sp_dns_ref ref, uint64_t now_ns) {
struct dns_mapping *mapping;
mapping = sp_dns_core_ref_get_mut(core, ref);
if (mapping == NULL || mapping->active_refs == 0)
return -1;
mapping->active_refs--;
if (mapping->active_refs == 0 && mapping->expires_ns != 0 &&
now_ns >= mapping->expires_ns)
memset(mapping, 0, sizeof(*mapping));
return 0;
}
_Static_assert(12U + SP_DNS_QUESTION_MAX + 16U <= SP_DNS_RESPONSE_MAX,
"a decoded DNS question and A answer must fit the response");
_Static_assert(SP_DNS_QUESTION_MAX <= sizeof(((struct sp_dns_query *)0)->question),
"a decoded DNS question must fit its storage");
static void sp_dns_encode_response_value(
const struct sp_dns_query *query, uint32_t answer_ip, int answer_a,
struct sp_dns_response *out) {
struct sp_dns_response candidate;
size_t length;
uint16_t flags;
if (answer_a)
assert(answer_ip != 0);
length = 12U + query->question_len + (answer_a ? 16U : 0U);
memset(&candidate, 0, sizeof(candidate));
flags = (uint16_t)(0x8000U | 0x0080U |
(query->request_flags & 0x0100U));
dns_put_u16(candidate.bytes, query->identifier);
dns_put_u16(candidate.bytes + 2, flags);
dns_put_u16(candidate.bytes + 4, 1);
dns_put_u16(candidate.bytes + 6, answer_a ? 1 : 0);
memcpy(candidate.bytes + 12, query->question, query->question_len);
if (answer_a) {
size_t off = 12U + query->question_len;
candidate.bytes[off++] = 0xc0;
candidate.bytes[off++] = 0x0c;
dns_put_u16(candidate.bytes + off, 1);
off += 2;
dns_put_u16(candidate.bytes + off, 1);
off += 2;
dns_put_u32(candidate.bytes + off, SP_DNS_TTL_SECONDS);
off += 4;
dns_put_u16(candidate.bytes + off, 4);
off += 2;
dns_put_u32(candidate.bytes + off, ntohl(answer_ip));
}
candidate.len = (uint16_t)length;
*out = candidate;
}
SP_TEST_API static enum sp_dns_effect_kind
sp_dns_core_step(struct sp_dns_core *core,
const struct sp_dns_core_input *input,
struct sp_dns_core_output *out) {
struct sp_dns_core_output candidate = {0};
enum sp_dns_effect_kind kind = SP_DNS_EFFECT_DROP;
int answer_a = 0;
uint32_t answer_ip = 0;
/* Only successfully decoded queries cross this internal boundary. */
assert(input->query.name_len != 0);
assert(input->query.name_len <= SP_DNS_NAME_MAX);
assert(input->query.name[input->query.name_len] == '\0');
assert(input->query.question_len >= 4);
assert(input->query.question_len <= SP_DNS_QUESTION_MAX);
assert((input->query.request_flags & 0x8000U) == 0);
if (input->credit == SP_DNS_CREDIT_NO_OUTPUT) {
kind = SP_DNS_EFFECT_RETRY;
goto commit;
}
if (input->credit != SP_DNS_CREDIT_AVAILABLE)
goto commit;
if (input->query.qclass == 1 && input->query.qtype == 1) {
/* Preserve the legacy NODATA response on table exhaustion. */
if (sp_dns_core_get_or_create(
core, input->query.name, input->query.name_len, input->now_ns,
&answer_ip) < 0)
candidate.mapping_exhausted = 1;
else
answer_a = 1;
}
sp_dns_encode_response_value(&input->query, answer_ip, answer_a, &candidate.response);
kind = SP_DNS_EFFECT_RESPONSE;
commit:
*out = candidate;
return kind;
}
static int dns_is_synthetic_ip(uint32_t ip_be) {
return (ntohl(ip_be) & 0xffff0000U) == DNS_SYNTHETIC_BASE;
}
static struct publish_udp_flow *publish_udp_find(uint16_t synthetic_port,
uint16_t child_port);
static void publish_udp_close_flow(struct publish_udp_flow *f);
static void publish_udp_update_rule_events(int rule_index);
static int publish_udp_send_now(const struct publish_rule *rule,
const struct publish_udp_flow *f,
const uint8_t *payload, size_t payload_len) {
struct iovec iov = {.iov_base = (void *)payload, .iov_len = payload_len};
char control[CMSG_SPACE(sizeof(struct in_pktinfo))];
struct msghdr msg;
struct cmsghdr *cmsg;
struct in_pktinfo pktinfo;
unsigned int interrupted = 0;
memset(control, 0, sizeof(control));
memset(&msg, 0, sizeof(msg));
memset(&pktinfo, 0, sizeof(pktinfo));
msg.msg_name = (void *)&f->host_peer;
msg.msg_namelen = sizeof(f->host_peer);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = control;
msg.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = IPPROTO_IP;
cmsg->cmsg_type = IP_PKTINFO;
cmsg->cmsg_len = CMSG_LEN(sizeof(pktinfo));
pktinfo.ipi_spec_dst.s_addr = f->published_local_ip;
pktinfo.ipi_addr.s_addr = f->published_local_ip;
memcpy(CMSG_DATA(cmsg), &pktinfo, sizeof(pktinfo));
for (;;) {
ssize_t n = sendmsg(rule->listen_fd, &msg, MSG_NOSIGNAL);
sp_resource_count_host_io(1, 1, n);
if (n == (ssize_t)payload_len)
return 0;
if (n >= 0)
return -1;
if (errno == EINTR && interrupted++ < 4)
continue;
if (errno == EINTR)
return 1;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return 1;
if (errno == ENOBUFS || errno == ENOMEM) {
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
return SP_UDP_SEND_DROPPED;
}
return -1;
}
}
static int publish_udp_queue_pending(struct publish_udp_flow *f,
const uint8_t *data, size_t len) {
int rc = sp_udp_pending_store(&f->core, f->generation,
(uint16_t)(f - publish_udp_flows), 1, NULL,
data, len);
if (rc == 0)
publish_udp_update_rule_events(f->rule_index);
return rc == 2 ? 1 : rc;
}
static void handle_dns_proxy(int tunfd, uint32_t src_ip, uint32_t dst_ip,
uint16_t src_port, uint16_t dst_port,
const uint8_t *payload, size_t payload_len) {
struct sp_dns_core_input input;
struct sp_dns_core_output output;
if (sp_dns_decode_query(payload, payload_len, &input.query) < 0) {
DBG("DNS proxy dropped malformed query or response");
return;
}
input.now_ns = sp_ns_now_ns();
input.credit = SP_DNS_CREDIT_AVAILABLE;
if (sp_dns_core_step(&g_dns_core, &input, &output) !=
SP_DNS_EFFECT_RESPONSE) {
DBG("DNS proxy dropped malformed query or response");
return;
}
if (output.mapping_exhausted)
DBG("DNS synthetic mapping table full; returning NODATA");
else if (input.query.qclass == 1 && input.query.qtype == 28)
DBG("DNS proxy returning empty AAAA answer for %s", input.query.name);
else if (input.query.qclass != 1 || input.query.qtype != 1)
DBG("DNS proxy returning empty answer for %s type=%u class=%u",
input.query.name, (unsigned)input.query.qtype,
(unsigned)input.query.qclass);
sp_udp_emit_tun(tunfd, ntohl(dst_ip), ntohl(src_ip), dst_port, src_port,
output.response.bytes, output.response.len, "DNS inject");
}
static int handle_publish_udp_from_child(uint32_t src_ip, uint32_t dst_ip,
uint16_t src_port, uint16_t dst_port,
const uint8_t *payload,
size_t payload_len) {
struct publish_udp_flow *f;
struct publish_rule *rule;
if (dst_ip != PUBLISH_SYNTHETIC_IP)
return 0;
if (src_ip != SANDBOX_IP)
return 1;
f = publish_udp_find(dst_port, src_port);
if (f == NULL)
return 1;
if (f->rule_index < 0 || f->rule_index >= publish_rule_count)
return 1;
rule = &publish_rules[f->rule_index];
if (rule->proto != IPPROTO_UDP || rule->listen_fd < 0 ||
rule->container_port != src_port)
return 1;
{
int rc = publish_udp_send_now(rule, f, payload, payload_len);
if (rc < 0) {
publish_udp_close_flow(f);
return 1;
}
if (rc == SP_UDP_SEND_RETAIN &&
publish_udp_queue_pending(f, payload, payload_len) < 0) {
publish_udp_close_flow(f);
return 1;
}
if (rc == 0) {
uint64_t now_ns = sp_ns_now_ns();
(void)sp_udp_runtime_step(&f->core, SP_UDP_PROGRESS, now_ns, 0);
}
return 1;
}
}
static void handle_udp(int tunfd, const uint8_t *pkt, size_t len,
const struct sp_packet_value *parsed) {
uint32_t src_ip;
uint32_t dst_ip;
uint16_t sport;
uint16_t dport;
const uint8_t *payload;
size_t plen;
if (sp_udp_dispatch_values(pkt, len, parsed, &src_ip, &dst_ip, &sport,
&dport, &payload, &plen) < 0)
return;
if (handle_publish_udp_from_child(src_ip, dst_ip, sport, dport, payload,
plen))
return;
if (g_net_policy.egress == EGRESS_SOCKS &&
g_net_policy.socks_remote_dns &&
dst_ip == DNS_PROXY_IP && dport == 53) {
handle_dns_proxy(tunfd, src_ip, dst_ip, sport, dport, payload, plen);
return;
}
/* Check for host gateway access (10.0.1.x -> 127.0.0.x) */
uint32_t target_ip = dst_ip;
int is_gateway = is_gateway_ip(target_ip);
if (g_net_policy.egress == EGRESS_NONE) {
DBG("[parent] UDP egress blocked by --egress=none: %s:%u",
inet_ntoa((struct in_addr){.s_addr = target_ip}), dport);
return;
}
if (is_gateway) {
if (!sp_policy_gateway_allowed(&g_net_policy, target_ip, dport,
IPPROTO_UDP)) {
DBG("[parent] UDP to 10.0.1.%d:%d blocked", gateway_last_octet(target_ip),
dport);
return;
}
target_ip = gateway_to_localhost(dst_ip);
DBG("[parent] UDP gateway: 10.0.1.%d:%d -> 127.0.0.%d:%d",
gateway_last_octet(dst_ip), dport, gateway_last_octet(dst_ip),
dport);
} else if (g_net_policy.egress == EGRESS_DIRECT &&
!sp_policy_direct_egress_allowed(&g_net_policy, target_ip, dport,
IPPROTO_UDP)) {
log_direct_egress_block(target_ip, dport, IPPROTO_UDP);
return;
}
if (g_net_policy.egress == EGRESS_SOCKS && !is_gateway) {
/* Find or create persistent UDP flow */
struct udp_flow *f = udp_find(src_ip, sport, dst_ip, dport,
SP_UDP_ROLE_SOCKS);
if (!f) {
struct sp_udp_key key;
if (sp_udp_admit_source(src_ip, SP_UDP_ROLE_SOCKS) < 0) {
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
if (sp_udp_admit_global() < 0) {
sp_resource_admission_release_source(
src_ip, sp_udp_admission_role(SP_UDP_ROLE_SOCKS));
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
f = udp_alloc(1);
if (f == NULL) {
sp_resource_admission_release_source(
src_ip, sp_udp_admission_role(SP_UDP_ROLE_SOCKS));
sp_resource_admission_release_source(0, SP_UDP_ADMISSION_GLOBAL);
return;
}
udp_flow_begin(f, SP_UDP_ROLE_SOCKS);
f->admission_held = 1;
f->global_admission_held = 1;
f->cli_ip = src_ip;
f->cli_port = sport;
f->srv_ip = dst_ip;
f->srv_port = dport;
if (g_net_policy.socks_remote_dns && dns_is_synthetic_ip(f->srv_ip)) {
struct sp_dns_ref mapping;
if (sp_dns_core_acquire_ip(&g_dns_core, f->srv_ip,
sp_ns_now_ns(), &mapping) < 0) {
udp_close_flow(f);
return;
}
f->dns_mapping_ref = mapping;
}
if (udp_socks_setup(f, &socks_proxy) < 0) {
udp_close_flow(f);
return;
}
sp_udp_outbound_key(&key, f->core.role, f->cli_ip, f->cli_port, f->srv_ip,
f->srv_port);
if (sp_udp_table_insert(&key, f->generation,
(uint16_t)(f - udp_flows), f->core.role) < 0) {
udp_close_flow(f);
return;
}
epoll_set_udp(f, EPOLL_CTL_ADD, FD_UDP_CTRL, udp_ctrl_events(f));
}
memcpy(f->last_quote, pkt, sizeof(f->last_quote));
f->has_last_quote = 1;
if (f->udp_relay >= 0) {
int src = udp_send_socks_now(f, payload, plen);
if (src < 0) {
udp_close_flow(f);
return;
}
if (src == SP_UDP_SEND_RETAIN &&
udp_queue_pending(f, payload, plen, pkt) < 0) {
udp_close_flow(f);
return;
}
if (src == SP_UDP_SEND_EMSGSIZE || src == SP_UDP_SEND_REFUSED)
(void)sp_udp_emit_error(tunfd, pkt, f,
(enum sp_udp_send_result)src);
udp_update_events(f);
} else {
int qrc = udp_queue_pending(f, payload, plen, pkt);
if (qrc < 0) {
udp_close_flow(f);
return;
}
}
} else {
struct udp_flow *f = udp_find(src_ip, sport, dst_ip, dport,
SP_UDP_ROLE_DIRECT);
if (!f) {
struct sp_udp_key key;
if (sp_udp_admit_source(src_ip, SP_UDP_ROLE_DIRECT) < 0) {
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
if (sp_udp_admit_global() < 0) {
sp_resource_admission_release_source(
src_ip, sp_udp_admission_role(SP_UDP_ROLE_DIRECT));
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
f = udp_alloc(0);
if (f == NULL) {
sp_resource_admission_release_source(
src_ip, sp_udp_admission_role(SP_UDP_ROLE_DIRECT));
sp_resource_admission_release_source(0, SP_UDP_ADMISSION_GLOBAL);
return;
}
udp_flow_begin(f, SP_UDP_ROLE_DIRECT);
f->admission_held = 1;
f->global_admission_held = 1;
f->cli_ip = src_ip;
f->cli_port = sport;
f->srv_ip = dst_ip;
f->srv_port = dport;
if (udp_direct_setup(f, target_ip, dport) < 0) {
udp_close_flow(f);
return;
}
sp_udp_outbound_key(&key, f->core.role, f->cli_ip, f->cli_port, f->srv_ip,
f->srv_port);
if (sp_udp_table_insert(&key, f->generation,
(uint16_t)(f - udp_flows), f->core.role) < 0) {
udp_close_flow(f);
return;
}
epoll_set_udp(f, EPOLL_CTL_ADD, FD_UDP_RELAY, EPOLLIN | EPOLLRDHUP | EPOLLERR | EPOLLHUP);
udp_update_events(f);
}
memcpy(f->last_quote, pkt, sizeof(f->last_quote));
f->has_last_quote = 1;
int src = udp_send_direct_now(f, payload, plen);
if (src < 0) {
udp_close_flow(f);
return;
}
if (src == SP_UDP_SEND_RETAIN &&
udp_queue_pending(f, payload, plen, pkt) < 0) {
udp_close_flow(f);
return;
}
if (src == SP_UDP_SEND_EMSGSIZE || src == SP_UDP_SEND_REFUSED)
(void)sp_udp_emit_error(tunfd, pkt, f,
(enum sp_udp_send_result)src);
udp_update_events(f);
DBG("UDP queued %zu bytes to %s:%d", plen,
is_gateway ? "127.0.0.1" : "remote", dport);
}
}
static uint32_t checksum_add_bytes(const void *buf, size_t len) {
const uint8_t *p = buf;
uint32_t sum = 0;
while (len > 1) {
sum += ((uint32_t)p[0] << 8) | (uint32_t)p[1];
p += 2;
len -= 2;
}
if (len > 0)
sum += (uint32_t)p[0] << 8;
return sum;
}
static uint16_t checksum_finish(uint32_t sum) {
while (sum >> 16)
sum = (sum & 0xffffU) + (sum >> 16);
return (uint16_t)~sum;
}
/* Bytewise packet codec. Parsed values contain only scalars and
* offsets: no object below retains a pointer into mutable TUN storage. */
enum sp_codec_status {
SP_CODEC_OK = 0,
SP_CODEC_SHORT,
SP_CODEC_VERSION,
SP_CODEC_LENGTH,
SP_CODEC_IHL,
SP_CODEC_FRAGMENT,
SP_CODEC_TTL,
SP_CODEC_CHECKSUM,
SP_CODEC_PROTOCOL,
SP_CODEC_PORT,
SP_CODEC_TCP_OFFSET,
SP_CODEC_TCP_FLAGS,
SP_CODEC_TCP_OPTIONS,
SP_CODEC_MSS,
SP_CODEC_SOURCE,
SP_CODEC_VNET_FLAGS,
SP_CODEC_VNET_TYPE,
SP_CODEC_VNET_FIELDS,
SP_CODEC_GSO_LIMIT,
SP_CODEC_OUTPUT,
};
enum {
SP_VNET_F_NEEDS_CSUM = 1,
SP_VNET_F_DATA_VALID = 2,
SP_VNET_GSO_NONE = 0,
SP_VNET_GSO_TCPV4 = 1,
SP_TCP_FLAG_FIN = 0x01,
SP_TCP_FLAG_SYN = 0x02,
SP_TCP_FLAG_RST = 0x04,
SP_TCP_FLAG_PSH = 0x08,
SP_TCP_FLAG_ACK = 0x10,
SP_TCP_FLAG_URG = 0x20,
SP_TCP_FLAG_ECE = 0x40,
SP_TCP_FLAG_CWR = 0x80,
};
struct sp_vnet_value {
uint8_t flags;
uint8_t gso_type;
uint16_t hdr_len;
uint16_t gso_size;
uint16_t csum_start;
uint16_t csum_offset;
};
struct sp_ipv4_value {
uint8_t ihl;
uint8_t tos;
uint16_t total_len;
uint16_t id;
uint16_t frag_off;
uint8_t ttl;
uint8_t protocol;
uint32_t src;
uint32_t dst;
size_t transport_offset;
size_t transport_len;
};
struct sp_tcp_value {
uint16_t src_port;
uint16_t dst_port;
uint32_t seq;
uint32_t ack;
uint8_t data_offset;
uint8_t flags;
uint16_t window;
uint16_t urgent;
size_t payload_offset;
size_t payload_len;
uint16_t mss;
uint8_t window_scale;
uint8_t has_mss;
uint8_t has_window_scale;
};
struct sp_udp_value {
uint16_t src_port;
uint16_t dst_port;
uint16_t length;
uint16_t checksum;
size_t payload_offset;
size_t payload_len;
};
struct sp_packet_value {
struct sp_vnet_value vnet;
struct sp_ipv4_value ip;
struct sp_tcp_value tcp;
struct sp_udp_value udp;
size_t packet_offset;
};
struct sp_scalar_packet {
size_t len;
uint8_t bytes[1500];
};
/* A control response quotes the original IPv4/UDP headers. The quote is an
* internal, already validated product; the pure builder copies it into the
* canonical ICMP packet and never parses caller-owned bytes. */
struct sp_icmp_control_request {
uint8_t code;
uint16_t mtu;
uint8_t quote[SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES];
};
_Static_assert(sizeof(struct virtio_net_hdr) == SP_VNET_HDR_BYTES,
"virtio-net header must match the negotiated 10-byte form");
static uint16_t sp_get_be16(const uint8_t *p) {
return (uint16_t)(((uint16_t)p[0] << 8) | (uint16_t)p[1]);
}
static uint32_t sp_get_be32(const uint8_t *p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
((uint32_t)p[2] << 8) | (uint32_t)p[3];
}
static uint16_t sp_get_le16(const uint8_t *p) {
return (uint16_t)((uint16_t)p[0] | ((uint16_t)p[1] << 8));
}
static void sp_put_be16(uint8_t *p, uint16_t value) {
p[0] = (uint8_t)(value >> 8);
p[1] = (uint8_t)value;
}
static void sp_put_be32(uint8_t *p, uint32_t value) {
p[0] = (uint8_t)(value >> 24);
p[1] = (uint8_t)(value >> 16);
p[2] = (uint8_t)(value >> 8);
p[3] = (uint8_t)value;
}
static void sp_put_le16(uint8_t *p, uint16_t value) {
p[0] = (uint8_t)value;
p[1] = (uint8_t)(value >> 8);
}
static int sp_slice_ok(size_t len, size_t offset, size_t need) {
return offset <= len && need <= len - offset;
}
static uint32_t sp_checksum_pseudo(uint32_t src, uint32_t dst, uint8_t proto,
uint16_t length) {
uint8_t pseudo[12];
sp_put_be32(pseudo, src);
sp_put_be32(pseudo + 4, dst);
pseudo[8] = 0;
pseudo[9] = proto;
sp_put_be16(pseudo + 10, length);
return checksum_add_bytes(pseudo, sizeof(pseudo));
}
static uint16_t sp_checksum_complete(const uint8_t *bytes, size_t len) {
return checksum_finish(checksum_add_bytes(bytes, len));
}
static uint16_t sp_checksum_transport(uint32_t src, uint32_t dst,
uint8_t proto, const uint8_t *segment,
size_t length) {
uint32_t sum;
if (length > UINT16_MAX)
return 0;
sum = sp_checksum_pseudo(src, dst, proto, (uint16_t)length);
sum += checksum_add_bytes(segment, length);
return checksum_finish(sum);
}
static uint16_t sp_checksum_partial_seed(uint32_t src, uint32_t dst,
uint8_t proto, uint16_t length) {
uint32_t sum = sp_checksum_pseudo(src, dst, proto, length);
while (sum >> 16)
sum = (sum & 0xffffU) + (sum >> 16);
return (uint16_t)sum;
}
static void sp_write_vnet(uint8_t out[SP_VNET_HDR_BYTES],
const struct sp_vnet_value *vnet) {
out[0] = vnet->flags;
out[1] = vnet->gso_type;
sp_put_le16(out + 2, vnet->hdr_len);
sp_put_le16(out + 4, vnet->gso_size);
sp_put_le16(out + 6, vnet->csum_start);
sp_put_le16(out + 8, vnet->csum_offset);
}
static enum sp_codec_status
sp_parse_vnet(const uint8_t *record, size_t len, struct sp_vnet_value *out) {
if (len < SP_VNET_HDR_BYTES)
return SP_CODEC_SHORT;
out->flags = record[0];
out->gso_type = record[1];
out->hdr_len = sp_get_le16(record + 2);
out->gso_size = sp_get_le16(record + 4);
out->csum_start = sp_get_le16(record + 6);
out->csum_offset = sp_get_le16(record + 8);
if ((out->flags & (uint8_t)~(SP_VNET_F_NEEDS_CSUM |
SP_VNET_F_DATA_VALID)) != 0 ||
out->flags == (SP_VNET_F_NEEDS_CSUM | SP_VNET_F_DATA_VALID))
return SP_CODEC_VNET_FLAGS;
if (out->gso_type != SP_VNET_GSO_NONE &&
out->gso_type != SP_VNET_GSO_TCPV4)
return SP_CODEC_VNET_TYPE;
return SP_CODEC_OK;
}
static enum sp_codec_status sp_parse_ipv4(const uint8_t *packet, size_t len,
int scalar,
struct sp_ipv4_value *out) {
uint8_t version;
size_t ihl_bytes;
if (len < SP_IPV4_HEADER_BYTES)
return SP_CODEC_SHORT;
version = packet[0] >> 4;
out->ihl = packet[0] & 0x0fU;
if (version != 4)
return SP_CODEC_VERSION;
if (out->ihl < 5)
return SP_CODEC_IHL;
ihl_bytes = (size_t)out->ihl * 4U;
if (!sp_slice_ok(len, 0, ihl_bytes))
return SP_CODEC_SHORT;
if (out->ihl != 5)
return SP_CODEC_IHL;
out->tos = packet[1];
out->total_len = sp_get_be16(packet + 2);
out->id = sp_get_be16(packet + 4);
out->frag_off = sp_get_be16(packet + 6);
out->ttl = packet[8];
out->protocol = packet[9];
out->src = sp_get_be32(packet + 12);
out->dst = sp_get_be32(packet + 16);
if ((size_t)out->total_len != len || out->total_len < ihl_bytes)
return SP_CODEC_LENGTH;
if (scalar && out->total_len > 1500)
return SP_CODEC_LENGTH;
if (out->ttl == 0)
return SP_CODEC_TTL;
if ((out->frag_off & (IP_MF | IP_OFFMASK)) != 0)
return SP_CODEC_FRAGMENT;
if (sp_checksum_complete(packet, ihl_bytes) != 0)
return SP_CODEC_CHECKSUM;
out->transport_offset = ihl_bytes;
out->transport_len = len - ihl_bytes;
return SP_CODEC_OK;
}
static enum sp_codec_status sp_parse_tcp_options(const uint8_t *tcp,
size_t header_len,
uint8_t flags,
struct sp_tcp_value *out) {
size_t pos = 20;
int after_eol = 0;
while (pos < header_len) {
uint8_t kind = tcp[pos];
if (after_eol) {
if (kind != 0)
return SP_CODEC_TCP_OPTIONS;
pos++;
continue;
}
if (kind == 0) {
after_eol = 1;
pos++;
continue;
}
if (kind == 1) {
pos++;
continue;
}
if (!sp_slice_ok(header_len, pos, 2) || tcp[pos + 1] < 2 ||
!sp_slice_ok(header_len, pos, tcp[pos + 1]))
return SP_CODEC_TCP_OPTIONS;
if (kind == 2) {
if ((flags & SP_TCP_FLAG_SYN) == 0 || tcp[pos + 1] != 4 || out->has_mss)
return SP_CODEC_TCP_OPTIONS;
out->mss = sp_get_be16(tcp + pos + 2);
if (out->mss < SP_TCP_MIN_PEER_MSS)
return SP_CODEC_MSS;
out->has_mss = 1;
} else if (kind == 3) {
if ((flags & SP_TCP_FLAG_SYN) == 0 || tcp[pos + 1] != 3 ||
out->has_window_scale)
return SP_CODEC_TCP_OPTIONS;
out->window_scale = tcp[pos + 2];
out->has_window_scale = 1;
}
pos += tcp[pos + 1];
}
return SP_CODEC_OK;
}
static enum sp_codec_status sp_parse_tcp(const uint8_t *packet, size_t len,
const struct sp_ipv4_value *ip,
struct sp_tcp_value *out) {
const uint8_t *tcp;
size_t header_len;
enum sp_codec_status status;
if (!sp_slice_ok(len, ip->transport_offset, ip->transport_len) ||
ip->transport_len < 20)
return SP_CODEC_SHORT;
tcp = packet + ip->transport_offset;
memset(out, 0, sizeof(*out));
out->src_port = sp_get_be16(tcp);
out->dst_port = sp_get_be16(tcp + 2);
out->seq = sp_get_be32(tcp + 4);
out->ack = sp_get_be32(tcp + 8);
out->data_offset = tcp[12] >> 4;
out->flags = tcp[13];
out->window = sp_get_be16(tcp + 14);
out->urgent = sp_get_be16(tcp + 18);
if (out->src_port == 0 || out->dst_port == 0)
return SP_CODEC_PORT;
if (out->data_offset < 5 || out->data_offset > 15)
return SP_CODEC_TCP_OFFSET;
header_len = (size_t)out->data_offset * 4U;
if (header_len > ip->transport_len)
return SP_CODEC_TCP_OFFSET;
if ((tcp[12] & 0x0fU) != 0 ||
(out->flags & (SP_TCP_FLAG_ECE | SP_TCP_FLAG_CWR)) != 0)
return SP_CODEC_TCP_FLAGS;
if (((out->flags & (SP_TCP_FLAG_SYN | SP_TCP_FLAG_FIN)) ==
(SP_TCP_FLAG_SYN | SP_TCP_FLAG_FIN)) ||
((out->flags & (SP_TCP_FLAG_SYN | SP_TCP_FLAG_RST)) ==
(SP_TCP_FLAG_SYN | SP_TCP_FLAG_RST)) ||
((out->flags & (SP_TCP_FLAG_FIN | SP_TCP_FLAG_RST)) ==
(SP_TCP_FLAG_FIN | SP_TCP_FLAG_RST)))
return SP_CODEC_TCP_FLAGS;
out->payload_offset = ip->transport_offset + header_len;
out->payload_len = ip->transport_len - header_len;
if ((out->flags & SP_TCP_FLAG_SYN) != 0 && out->payload_len != 0)
return SP_CODEC_TCP_FLAGS;
if ((out->flags & SP_TCP_FLAG_RST) != 0 && out->payload_len != 0)
return SP_CODEC_TCP_FLAGS;
status = sp_parse_tcp_options(tcp, header_len, out->flags, out);
if (status != SP_CODEC_OK)
return status;
if (sp_checksum_transport(ip->src, ip->dst, IPPROTO_TCP, tcp,
ip->transport_len) != 0)
return SP_CODEC_CHECKSUM;
return SP_CODEC_OK;
}
static enum sp_codec_status sp_parse_udp(const uint8_t *packet, size_t len,
const struct sp_ipv4_value *ip,
struct sp_udp_value *out) {
const uint8_t *udp;
if (!sp_slice_ok(len, ip->transport_offset, ip->transport_len) ||
ip->transport_len < SP_UDP_HEADER_BYTES)
return SP_CODEC_SHORT;
udp = packet + ip->transport_offset;
out->src_port = sp_get_be16(udp);
out->dst_port = sp_get_be16(udp + 2);
out->length = sp_get_be16(udp + 4);
out->checksum = sp_get_be16(udp + 6);
if (out->src_port == 0 || out->dst_port == 0)
return SP_CODEC_PORT;
if (out->length != ip->transport_len || out->length < SP_UDP_HEADER_BYTES)
return SP_CODEC_LENGTH;
out->payload_offset = ip->transport_offset + SP_UDP_HEADER_BYTES;
out->payload_len = out->length - SP_UDP_HEADER_BYTES;
if (out->payload_len > SP_UDP_PAYLOAD_MAX)
return SP_CODEC_LENGTH;
if (out->checksum != 0 &&
sp_checksum_transport(ip->src, ip->dst, IPPROTO_UDP, udp,
out->length) != 0)
return SP_CODEC_CHECKSUM;
return SP_CODEC_OK;
}
static enum sp_codec_status
sp_parse_ipv4_packet(const uint8_t *packet, size_t len, int scalar,
int require_child_source, struct sp_packet_value *out) {
enum sp_codec_status status;
memset(out, 0, sizeof(*out));
status = sp_parse_ipv4(packet, len, scalar, &out->ip);
if (status != SP_CODEC_OK)
return status;
if (require_child_source && out->ip.src != ntohl(SANDBOX_IP))
return SP_CODEC_SOURCE;
if (out->ip.protocol == IPPROTO_TCP)
return sp_parse_tcp(packet, len, &out->ip, &out->tcp);
if (out->ip.protocol == IPPROTO_UDP)
return sp_parse_udp(packet, len, &out->ip, &out->udp);
return SP_CODEC_PROTOCOL;
}
static enum sp_codec_status
sp_parse_tun_frame_source(const uint8_t *record, size_t len,
int require_child_source,
struct sp_packet_value *out) {
enum sp_codec_status status;
const uint8_t *packet;
size_t packet_len;
size_t header_end;
size_t segments;
uint16_t checksum;
memset(out, 0, sizeof(*out));
status = sp_parse_vnet(record, len, &out->vnet);
if (status != SP_CODEC_OK)
return status;
packet = record + SP_VNET_HDR_BYTES;
packet_len = len - SP_VNET_HDR_BYTES;
out->packet_offset = SP_VNET_HDR_BYTES;
status = sp_parse_ipv4(packet, packet_len,
out->vnet.gso_type == SP_VNET_GSO_NONE, &out->ip);
if (status != SP_CODEC_OK)
return status;
if (require_child_source && out->ip.src != ntohl(SANDBOX_IP))
return SP_CODEC_SOURCE;
if (out->ip.protocol == IPPROTO_TCP)
status = sp_parse_tcp(packet, packet_len, &out->ip, &out->tcp);
else if (out->ip.protocol == IPPROTO_UDP)
status = sp_parse_udp(packet, packet_len, &out->ip, &out->udp);
else
status = SP_CODEC_PROTOCOL;
if (status != SP_CODEC_OK && status != SP_CODEC_CHECKSUM)
return status;
header_end = out->ip.transport_offset;
if (out->ip.protocol == IPPROTO_TCP)
header_end = out->tcp.payload_offset;
else if (out->ip.protocol == IPPROTO_UDP)
header_end = out->udp.payload_offset;
if (out->vnet.hdr_len > packet_len)
return SP_CODEC_VNET_FIELDS;
if (out->vnet.gso_type == SP_VNET_GSO_NONE) {
if (out->vnet.gso_size != 0 || out->vnet.hdr_len != 0)
return SP_CODEC_VNET_FIELDS;
if (out->vnet.flags == 0 || out->vnet.flags == SP_VNET_F_DATA_VALID) {
if (out->vnet.csum_start != 0 || out->vnet.csum_offset != 0)
return SP_CODEC_VNET_FIELDS;
return status;
}
if (out->vnet.flags != SP_VNET_F_NEEDS_CSUM ||
(out->ip.protocol != IPPROTO_TCP &&
(out->ip.protocol != IPPROTO_UDP || out->udp.checksum == 0)))
return SP_CODEC_VNET_FLAGS;
} else {
if (out->vnet.flags != SP_VNET_F_NEEDS_CSUM || out->vnet.gso_size == 0)
return SP_CODEC_VNET_FIELDS;
if (out->vnet.gso_type == SP_VNET_GSO_TCPV4 &&
out->ip.protocol != IPPROTO_TCP)
return SP_CODEC_PROTOCOL;
if (out->vnet.hdr_len != header_end &&
out->vnet.hdr_len != header_end + out->vnet.gso_size)
return SP_CODEC_VNET_FIELDS;
segments = (packet_len - header_end + out->vnet.gso_size - 1U) /
out->vnet.gso_size;
if (segments == 0 || segments > SP_TCP_GSO_POLICY_SEGS)
return SP_CODEC_GSO_LIMIT;
if (out->vnet.gso_type == SP_VNET_GSO_TCPV4 &&
((out->tcp.flags & (SP_TCP_FLAG_SYN | SP_TCP_FLAG_RST |
SP_TCP_FLAG_URG | SP_TCP_FLAG_ECE |
SP_TCP_FLAG_CWR)) != 0 ||
out->vnet.gso_size > 1500U - header_end))
return SP_CODEC_TCP_FLAGS;
}
if (out->vnet.csum_start != out->ip.transport_offset)
return SP_CODEC_VNET_FIELDS;
if (out->ip.protocol == IPPROTO_TCP) {
if (out->vnet.csum_offset != 16)
return SP_CODEC_VNET_FIELDS;
checksum = sp_get_be16(packet + out->ip.transport_offset + 16);
} else if (out->ip.protocol == IPPROTO_UDP) {
if (out->vnet.csum_offset != 6)
return SP_CODEC_VNET_FIELDS;
checksum = sp_get_be16(packet + out->ip.transport_offset + 6);
} else {
return SP_CODEC_VNET_FIELDS;
}
if (checksum != sp_checksum_partial_seed(
out->ip.src, out->ip.dst, out->ip.protocol,
(uint16_t)out->ip.transport_len))
return SP_CODEC_CHECKSUM;
return SP_CODEC_OK;
}
static enum sp_codec_status
sp_parse_tun_frame(const uint8_t *record, size_t len,
struct sp_packet_value *out) {
return sp_parse_tun_frame_source(record, len, 1, out);
}
static int sp_vnet_output_envelope_valid(const uint8_t *record, size_t len) {
struct sp_vnet_value vnet;
struct sp_ipv4_value ip;
struct sp_packet_value parsed;
const uint8_t *packet;
size_t packet_len;
if (sp_parse_vnet(record, len, &vnet) != SP_CODEC_OK)
return 0;
packet = record + SP_VNET_HDR_BYTES;
packet_len = len - SP_VNET_HDR_BYTES;
if (sp_parse_ipv4(packet, packet_len, vnet.gso_type == SP_VNET_GSO_NONE,
&ip) != SP_CODEC_OK)
return 0;
if (vnet.gso_type == SP_VNET_GSO_NONE)
return vnet.flags == 0 && vnet.hdr_len == 0 && vnet.gso_size == 0 &&
vnet.csum_start == 0 && vnet.csum_offset == 0;
if (vnet.flags != SP_VNET_F_NEEDS_CSUM || vnet.gso_size == 0 ||
vnet.csum_start != ip.transport_offset)
return 0;
if (vnet.gso_type == SP_VNET_GSO_TCPV4) {
if (ip.protocol != IPPROTO_TCP ||
vnet.gso_size > SP_TCP_MTU1500_MAX_MSS ||
vnet.csum_offset != 16)
return 0;
return sp_parse_tun_frame_source(record, len, 0, &parsed) ==
SP_CODEC_OK;
}
return 0;
}
static void sp_rechecksum_ipv4(uint8_t *packet) {
sp_put_be16(packet + 10, 0);
sp_put_be16(packet + 10, sp_checksum_complete(packet, 20));
}
static void sp_rechecksum_tcp(uint8_t *packet, size_t packet_len) {
size_t tcp_len = packet_len - 20;
uint16_t checksum;
sp_put_be16(packet + 20 + 16, 0);
checksum = sp_checksum_transport(sp_get_be32(packet + 12),
sp_get_be32(packet + 16), IPPROTO_TCP,
packet + 20, tcp_len);
sp_put_be16(packet + 20 + 16, checksum);
}
static void sp_rechecksum_udp(uint8_t *packet, size_t packet_len) {
size_t udp_len = packet_len - SP_IPV4_HEADER_BYTES;
uint16_t checksum;
sp_put_be16(packet + SP_IPV4_HEADER_BYTES + 6, 0);
checksum = sp_checksum_transport(sp_get_be32(packet + 12),
sp_get_be32(packet + 16), IPPROTO_UDP,
packet + SP_IPV4_HEADER_BYTES, udp_len);
sp_put_be16(packet + SP_IPV4_HEADER_BYTES + 6,
checksum == 0 ? UINT16_MAX : checksum);
}
static int sp_udp_dispatch_values(const uint8_t *packet, size_t packet_len,
const struct sp_packet_value *parsed,
uint32_t *src_ip, uint32_t *dst_ip,
uint16_t *src_port, uint16_t *dst_port,
const uint8_t **payload,
size_t *payload_len) {
if (packet == NULL || parsed == NULL || src_ip == NULL || dst_ip == NULL ||
src_port == NULL || dst_port == NULL || payload == NULL ||
payload_len == NULL || parsed->ip.protocol != IPPROTO_UDP ||
parsed->ip.total_len != packet_len ||
parsed->ip.transport_offset != SP_IPV4_HEADER_BYTES ||
parsed->udp.length < SP_UDP_HEADER_BYTES ||
parsed->udp.length != parsed->ip.transport_len ||
parsed->udp.payload_offset !=
parsed->ip.transport_offset + SP_UDP_HEADER_BYTES ||
parsed->udp.payload_len !=
(size_t)parsed->udp.length - SP_UDP_HEADER_BYTES ||
!sp_slice_ok(packet_len, parsed->udp.payload_offset,
parsed->udp.payload_len))
return -1;
*src_ip = htonl(parsed->ip.src);
*dst_ip = htonl(parsed->ip.dst);
*src_port = parsed->udp.src_port;
*dst_port = parsed->udp.dst_port;
*payload = packet + parsed->udp.payload_offset;
*payload_len = parsed->udp.payload_len;
return 0;
}
SP_TEST_API static enum sp_codec_status
sp_build_udp_packet_value(uint32_t src_ip, uint32_t dst_ip, uint16_t src_port,
uint16_t dst_port, const uint8_t *payload,
size_t payload_len, struct sp_scalar_packet *out) {
size_t total_len = SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES + payload_len;
if (out == NULL || src_port == 0 || dst_port == 0 ||
(payload_len != 0 && payload == NULL))
return SP_CODEC_OUTPUT;
if (payload_len > SP_UDP_PAYLOAD_MAX || total_len > sizeof(out->bytes))
return SP_CODEC_LENGTH;
memset(out, 0, sizeof(*out));
out->len = total_len;
out->bytes[0] = 0x45;
sp_put_be16(out->bytes + 2, (uint16_t)total_len);
out->bytes[8] = 64;
out->bytes[9] = IPPROTO_UDP;
sp_put_be32(out->bytes + 12, src_ip);
sp_put_be32(out->bytes + 16, dst_ip);
sp_put_be16(out->bytes + SP_IPV4_HEADER_BYTES, src_port);
sp_put_be16(out->bytes + SP_IPV4_HEADER_BYTES + 2, dst_port);
sp_put_be16(out->bytes + SP_IPV4_HEADER_BYTES + 4,
(uint16_t)(SP_UDP_HEADER_BYTES + payload_len));
if (payload_len != 0)
memcpy(out->bytes + SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES, payload,
payload_len);
sp_rechecksum_ipv4(out->bytes);
return SP_CODEC_OK;
}
static void sp_udp_emit_tun(int tunfd, uint32_t src_ip, uint32_t dst_ip,
uint16_t src_port, uint16_t dst_port,
const uint8_t *payload, size_t payload_len,
const char *context) {
struct sp_scalar_packet packet;
if (sp_build_udp_packet_value(src_ip, dst_ip, src_port, dst_port, payload,
payload_len, &packet) != SP_CODEC_OK)
return;
IGNORE_RESULT(
tun_write_packet(tunfd, packet.bytes, packet.len, context));
}
static int sp_normalize_vnet_scalar_checksum(
uint8_t *packet, const struct sp_packet_value *parsed) {
if (parsed->vnet.gso_type != SP_VNET_GSO_NONE)
return -1;
if (parsed->vnet.flags != SP_VNET_F_NEEDS_CSUM)
return 0;
if (parsed->ip.protocol == IPPROTO_TCP) {
sp_rechecksum_tcp(packet, parsed->ip.total_len);
return 0;
}
if (parsed->ip.protocol == IPPROTO_UDP) {
sp_rechecksum_udp(packet, parsed->ip.total_len);
return 0;
}
return -1;
}
static enum sp_codec_status
sp_gso_expand_oracle_source(const uint8_t *record, size_t record_len,
struct sp_scalar_packet *out, size_t out_cap,
size_t *out_count, int require_child_source) {
struct sp_packet_value parsed;
enum sp_codec_status status;
const uint8_t *packet;
size_t header_len;
size_t payload_len;
size_t count;
size_t consumed = 0;
*out_count = 0;
status = sp_parse_tun_frame_source(record, record_len,
require_child_source, &parsed);
if (status != SP_CODEC_OK)
return status;
if (parsed.vnet.gso_type != SP_VNET_GSO_TCPV4)
return SP_CODEC_VNET_TYPE;
packet = record + SP_VNET_HDR_BYTES;
header_len = parsed.tcp.payload_offset;
payload_len = parsed.ip.total_len - header_len;
count = (payload_len + parsed.vnet.gso_size - 1U) / parsed.vnet.gso_size;
if (count == 0 || count > out_cap || count > SP_TCP_GSO_POLICY_SEGS)
return SP_CODEC_OUTPUT;
for (size_t i = 0; i < count; ++i) {
size_t part = payload_len - consumed;
uint8_t flags = parsed.tcp.flags;
if (part > parsed.vnet.gso_size)
part = parsed.vnet.gso_size;
out[i].len = header_len + part;
memcpy(out[i].bytes, packet, header_len);
memcpy(out[i].bytes + header_len, packet + header_len + consumed, part);
sp_put_be16(out[i].bytes + 2, (uint16_t)out[i].len);
sp_put_be16(out[i].bytes + 4, (uint16_t)(parsed.ip.id + i));
sp_put_be32(out[i].bytes + 20 + 4,
parsed.tcp.seq + (uint32_t)consumed);
if (i + 1U != count)
flags &= (uint8_t)~(SP_TCP_FLAG_FIN | SP_TCP_FLAG_PSH);
out[i].bytes[20 + 13] = flags;
sp_rechecksum_ipv4(out[i].bytes);
sp_rechecksum_tcp(out[i].bytes, out[i].len);
consumed += part;
}
if (consumed != payload_len)
return SP_CODEC_OUTPUT;
*out_count = count;
return SP_CODEC_OK;
}
SP_ORACLE_API static enum sp_codec_status
sp_gso_expand_oracle(const uint8_t *record, size_t record_len,
struct sp_scalar_packet *out, size_t out_cap,
size_t *out_count) {
return sp_gso_expand_oracle_source(record, record_len, out, out_cap,
out_count, 1);
}
static size_t sp_vnet_build_tcp_record(uint8_t *record, size_t capacity,
uint32_t src, uint32_t dst,
uint16_t src_port, uint16_t dst_port,
uint32_t seq, uint32_t ack, uint8_t flags,
const uint8_t *payload,
size_t payload_len, uint16_t gso_size) {
size_t packet_len = 40 + payload_len;
size_t record_len = SP_VNET_HDR_BYTES + packet_len;
uint8_t *packet;
if (record_len > capacity || packet_len > UINT16_MAX ||
(gso_size != 0 && payload_len <= gso_size))
return 0;
memset(record, 0, record_len);
packet = record + SP_VNET_HDR_BYTES;
packet[0] = 0x45;
sp_put_be16(packet + 2, (uint16_t)packet_len);
sp_put_be16(packet + 4, 1);
packet[8] = 64;
packet[9] = IPPROTO_TCP;
sp_put_be32(packet + 12, src);
sp_put_be32(packet + 16, dst);
sp_put_be16(packet + 20, src_port);
sp_put_be16(packet + 22, dst_port);
sp_put_be32(packet + 24, seq);
sp_put_be32(packet + 28, ack);
packet[32] = 0x50;
packet[33] = flags;
sp_put_be16(packet + 34, UINT16_MAX);
if (payload_len != 0)
memcpy(packet + 40, payload, payload_len);
sp_rechecksum_ipv4(packet);
if (gso_size == 0) {
sp_rechecksum_tcp(packet, packet_len);
} else {
struct sp_vnet_value vnet = {
.flags = SP_VNET_F_NEEDS_CSUM,
.gso_type = SP_VNET_GSO_TCPV4,
.hdr_len = 40,
.gso_size = gso_size,
.csum_start = 20,
.csum_offset = 16,
};
sp_write_vnet(record, &vnet);
sp_put_be16(packet + 36,
sp_checksum_partial_seed(src, dst, IPPROTO_TCP,
(uint16_t)(20 + payload_len)));
}
return record_len;
}
/* A scalar emit is a bounded view over one immutable effect and payload. */
struct sp_tcp_emit_cursor {
size_t payload_offset;
size_t segment_index;
size_t segment_count;
};
static int sp_tcp_emit_cursor_init(const struct sp_fx_tcp_emit_batch *effect,
size_t payload_len,
struct sp_tcp_emit_cursor *cursor) {
if (effect == NULL || cursor == NULL || effect->mss == 0 ||
effect->end.value < effect->start.value ||
payload_len != effect->end.value - effect->start.value ||
payload_len == 0 ||
payload_len > SIZE_MAX - (size_t)(effect->mss - 1U))
return -1;
cursor->payload_offset = 0;
cursor->segment_index = 0;
cursor->segment_count =
(payload_len + (size_t)effect->mss - 1U) / (size_t)effect->mss;
if (cursor->segment_count == 0 ||
cursor->segment_count > SP_TCP_CORE_MAX_SEGMENTS)
return -1;
return 0;
}
/* Format one scalar record; the caller commits the cursor only after the sink
* accepts the returned record. */
SP_TEST_API static size_t sp_tcp_emit_format_scalar(
const struct sp_fx_tcp_emit_batch *effect, uint32_t src, uint32_t dst,
uint16_t src_port, uint16_t dst_port, const uint8_t *payload,
size_t payload_len, struct sp_tcp_emit_cursor *cursor, uint8_t *record,
size_t capacity, size_t *segment_payload_len) {
size_t part;
if (effect == NULL || cursor == NULL || record == NULL ||
segment_payload_len == NULL ||
(payload == NULL && payload_len != 0) ||
cursor->segment_count == 0 ||
cursor->segment_count > SP_TCP_CORE_MAX_SEGMENTS ||
cursor->segment_index >= cursor->segment_count ||
cursor->payload_offset >= payload_len ||
effect->mss == 0 || effect->end.value < effect->start.value ||
payload_len != effect->end.value - effect->start.value)
return 0;
part = payload_len - cursor->payload_offset;
if (part > (size_t)effect->mss)
part = (size_t)effect->mss;
if (part == 0)
return 0;
uint8_t flags = (uint8_t)effect->flags;
if (cursor->segment_index + 1U != cursor->segment_count)
flags &= (uint8_t)~(SP_TCP_CORE_FIN | SP_TCP_CORE_PSH);
size_t record_len = sp_vnet_build_tcp_record(
record, capacity, src, dst, src_port, dst_port,
(uint32_t)(effect->seq.value + cursor->payload_offset),
(uint32_t)effect->ack.value, flags, payload + cursor->payload_offset,
part, 0);
if (record_len == 0)
return 0;
sp_put_be16(record + SP_VNET_HDR_BYTES + 4,
(uint16_t)(1U + cursor->segment_index));
sp_rechecksum_ipv4(record + SP_VNET_HDR_BYTES);
cursor->payload_offset += part;
cursor->segment_index++;
*segment_payload_len = part;
return record_len;
}
enum sp_egress_io_status {
SP_EGRESS_IO_PROGRESS = 0,
SP_EGRESS_IO_BLOCKED,
SP_EGRESS_IO_EOF,
SP_EGRESS_IO_FATAL,
};
struct sp_egress_peek_result {
enum sp_egress_io_status status;
size_t observed;
};
struct sp_egress_consume_result {
enum sp_egress_io_status status;
size_t consumed;
};
enum {
SP_EGRESS_PEEK_OFF_UNKNOWN = 0,
SP_EGRESS_PEEK_OFF_SUPPORTED,
SP_EGRESS_PEEK_OFF_UNAVAILABLE,
};
/*
* Plain TCP SO_PEEK_OFF support is a kernel capability, not a per-flow
* property. Linux 6.1 rejects it with EOPNOTSUPP, so remember that first
* result instead of repeating the rejected setsockopt for every observation
* and materialization. Forced fallback paths remain independent of the cache.
*/
static uint8_t g_egress_peek_off_capability;
#ifdef SOCKPUPPET_EGRESS_SYSCALL_HOOKS
typedef int (*sp_egress_setsockopt_fn)(int, int, int, const void *, socklen_t);
typedef ssize_t (*sp_egress_recvmsg_fn)(int, struct msghdr *, int);
static sp_egress_setsockopt_fn g_egress_setsockopt_fn;
static sp_egress_recvmsg_fn g_egress_recvmsg_fn;
static int sp_egress_setsockopt(int fd, int level, int option, const void *value,
socklen_t length) {
if (g_egress_setsockopt_fn != NULL)
return g_egress_setsockopt_fn(fd, level, option, value, length);
return setsockopt(fd, level, option, value, length);
}
static ssize_t sp_egress_recvmsg(int fd, struct msghdr *message, int flags) {
if (g_egress_recvmsg_fn != NULL)
return g_egress_recvmsg_fn(fd, message, flags);
return recvmsg(fd, message, flags);
}
#define SP_EGRESS_SETSOCKOPT(fd, level, option, value, length) \
sp_egress_setsockopt((fd), (level), (option), (value), (length))
#define SP_EGRESS_RECVMSG(fd, message, flags) \
sp_egress_recvmsg((fd), (message), (flags))
#else
#define SP_EGRESS_SETSOCKOPT(fd, level, option, value, length) \
setsockopt((fd), (level), (option), (value), (length))
#define SP_EGRESS_RECVMSG(fd, message, flags) recvmsg((fd), (message), (flags))
#endif
SP_TEST_API static struct sp_egress_peek_result
sp_egress_peek_backend(int fd, uint64_t stream_consumed, uint64_t stream_tail,
int read_half_closed, int force_fallback,
uint8_t *discard, size_t discard_capacity, uint8_t *target,
size_t target_capacity) {
struct sp_egress_peek_result result = {.status = SP_EGRESS_IO_FATAL};
uint64_t relative_u64;
size_t relative;
if (stream_tail < stream_consumed || target == NULL ||
target_capacity == 0) {
errno = EINVAL;
return result;
}
relative_u64 = stream_tail - stream_consumed;
if (relative_u64 > SIZE_MAX || relative_u64 > INT_MAX) {
errno = EOVERFLOW;
return result;
}
relative = (size_t)relative_u64;
#ifdef SO_PEEK_OFF
if (!force_fallback &&
g_egress_peek_off_capability != SP_EGRESS_PEEK_OFF_UNAVAILABLE) {
int offset = (int)relative;
if (SP_EGRESS_SETSOCKOPT(fd, SOL_SOCKET, SO_PEEK_OFF, &offset,
sizeof(offset)) == 0) {
g_egress_peek_off_capability = SP_EGRESS_PEEK_OFF_SUPPORTED;
ssize_t count = recv(fd, target, target_capacity,
MSG_PEEK | MSG_DONTWAIT);
int saved = errno;
int reset = -1;
if (SP_EGRESS_SETSOCKOPT(fd, SOL_SOCKET, SO_PEEK_OFF, &reset,
sizeof(reset)) < 0)
return result;
if (count > 0) {
result.status = SP_EGRESS_IO_PROGRESS;
result.observed = (size_t)count;
} else if (count == 0) {
result.status = SP_EGRESS_IO_EOF;
} else if (saved == EAGAIN || saved == EWOULDBLOCK) {
result.status = SP_EGRESS_IO_BLOCKED;
} else {
errno = saved;
}
return result;
}
if (errno != ENOPROTOOPT && errno != EOPNOTSUPP && errno != EINVAL)
return result;
g_egress_peek_off_capability = SP_EGRESS_PEEK_OFF_UNAVAILABLE;
DBG("A4 SO_PEEK_OFF unavailable; caching bounded fallback");
}
#else
(void)force_fallback;
#endif
if (relative > discard_capacity || (relative != 0 && discard == NULL)) {
errno = ENOBUFS;
return result;
}
struct iovec iov[2];
size_t iov_count = 0;
if (relative != 0) {
iov[iov_count].iov_base = discard;
iov[iov_count].iov_len = relative;
iov_count++;
}
iov[iov_count].iov_base = target;
iov[iov_count].iov_len = target_capacity;
iov_count++;
struct msghdr message = {.msg_iov = iov, .msg_iovlen = iov_count};
ssize_t count = SP_EGRESS_RECVMSG(fd, &message, MSG_PEEK | MSG_DONTWAIT);
if (count < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
result.status = SP_EGRESS_IO_BLOCKED;
return result;
}
if ((uint64_t)count < relative_u64) {
errno = EPROTO;
return result;
}
if ((uint64_t)count > relative_u64) {
result.status = SP_EGRESS_IO_PROGRESS;
result.observed = (size_t)((uint64_t)count - relative_u64);
} else if (relative == 0 || read_half_closed) {
result.status = SP_EGRESS_IO_EOF;
} else {
result.status = SP_EGRESS_IO_BLOCKED;
}
return result;
}
SP_TEST_API static struct sp_egress_consume_result
sp_egress_consume_backend(int fd, size_t authorized, int force_scratch,
uint8_t *scratch, size_t scratch_capacity) {
struct sp_egress_consume_result result = {.status = SP_EGRESS_IO_FATAL};
ssize_t count;
if (authorized == 0) {
errno = EINVAL;
return result;
}
if (!force_scratch) {
count = recv(fd, NULL, authorized, MSG_TRUNC | MSG_DONTWAIT);
if (count >= 0) {
if (count == 0)
result.status = SP_EGRESS_IO_EOF;
else {
result.status = SP_EGRESS_IO_PROGRESS;
result.consumed = (size_t)count;
}
return result;
}
if (errno == EAGAIN || errno == EWOULDBLOCK) {
result.status = SP_EGRESS_IO_BLOCKED;
return result;
}
if (errno != EFAULT && errno != EINVAL && errno != EOPNOTSUPP)
return result;
}
if (scratch == NULL || scratch_capacity == 0) {
errno = ENOBUFS;
return result;
}
count = recv(fd, scratch,
authorized < scratch_capacity ? authorized : scratch_capacity,
MSG_DONTWAIT);
if (count > 0) {
result.status = SP_EGRESS_IO_PROGRESS;
result.consumed = (size_t)count;
} else if (count == 0) {
result.status = SP_EGRESS_IO_EOF;
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
result.status = SP_EGRESS_IO_BLOCKED;
}
return result;
}
typedef ssize_t (*sp_tcp_write_fn)(void *context, const uint8_t *buf,
size_t length);
/* Kept as a named adapter type for the existing typed egress entry point and
* fixture callers; both directions use the same callback ABI. */
typedef sp_tcp_write_fn sp_egress_write_fn;
/* Classify one nonblocking stream write. The caller chooses whether a short
* positive prefix is valid: backend stream sends accept it, while a TUN frame
* must be all-or-nothing. No adapter owns or retains the caller's bytes. */
static enum sp_egress_io_status
sp_tcp_write_slice_with(sp_tcp_write_fn write_fn, void *context,
const uint8_t *buf, size_t length, int exact,
size_t *accepted) {
if (accepted == NULL) {
errno = EINVAL;
return SP_EGRESS_IO_FATAL;
}
*accepted = 0;
if (write_fn == NULL || buf == NULL || length == 0) {
errno = EINVAL;
return SP_EGRESS_IO_FATAL;
}
for (unsigned interrupts = 0; interrupts < 8; ++interrupts) {
ssize_t count = write_fn(context, buf, length);
if (count > 0) {
if ((uint64_t)count > length ||
(exact && (size_t)count != length)) {
errno = EIO;
return SP_EGRESS_IO_FATAL;
}
*accepted = (size_t)count;
return SP_EGRESS_IO_PROGRESS;
}
if (count == 0) {
errno = EIO;
return SP_EGRESS_IO_FATAL;
}
if (errno == EINTR)
continue;
if (errno == EAGAIN || errno == EWOULDBLOCK)
return SP_EGRESS_IO_BLOCKED;
return SP_EGRESS_IO_FATAL;
}
errno = EINTR;
return SP_EGRESS_IO_BLOCKED;
}
static enum sp_egress_io_status
sp_egress_write_tun_record_with(sp_egress_write_fn write_fn, void *context,
const uint8_t *record, size_t length) {
size_t accepted;
if (!sp_vnet_output_envelope_valid(record, length)) {
errno = EPROTO;
return SP_EGRESS_IO_FATAL;
}
return sp_tcp_write_slice_with(write_fn, context, record, length, 1,
&accepted);
}
SP_TEST_API static size_t sp_egress_build_tcp_emit_record(
uint8_t *record, size_t capacity, uint32_t src, uint32_t dst,
uint16_t src_port, uint16_t dst_port,
const struct sp_fx_tcp_emit_batch *effect,
const uint8_t *payload, size_t payload_len, size_t *logical_segments) {
struct sp_tcp_emit_cursor cursor;
uint16_t gso_size;
size_t record_len;
if (logical_segments == NULL ||
(payload == NULL && payload_len != 0) ||
sp_tcp_emit_cursor_init(effect, payload_len, &cursor) < 0)
return 0;
*logical_segments = cursor.segment_count;
gso_size = *logical_segments > 1 ? effect->mss : 0;
record_len = sp_vnet_build_tcp_record(
record, capacity, src, dst, src_port, dst_port,
(uint32_t)effect->seq.value, (uint32_t)effect->ack.value,
(uint8_t)effect->flags, payload, payload_len, gso_size);
return record_len;
}
struct sp_egress_executor_flow {
uint64_t flow;
uint64_t generation;
uint64_t stream_consumed;
uint64_t stream_tail;
uint64_t next_control_seq;
uint8_t tcp_backend;
uint8_t eof_reported;
};
static inline void sp_tcp_event_context_init(
struct sp_typed_event_context *context, uint64_t flow,
uint64_t generation, uint64_t now_ns) {
*context = (struct sp_typed_event_context){
.key = sp_flow_key_make((sp_flow_id){.value = flow},
(sp_generation_id){.value = generation}),
.now = {.value = now_ns},
};
}
static uint64_t sp_egress_min_u64(uint64_t left, uint64_t right) {
return left < right ? left : right;
}
static int sp_egress_arm_backend_lowat(int fd,
const struct sp_egress_executor_flow *flow) {
uint64_t relative;
int lowat;
if (flow->stream_tail < flow->stream_consumed)
return -1;
if (!flow->tcp_backend)
return 0;
relative = flow->stream_tail - flow->stream_consumed;
if (relative >= INT_MAX)
lowat = INT_MAX;
else
lowat = (int)relative + 1;
return setsockopt(fd, SOL_SOCKET, SO_RCVLOWAT, &lowat, sizeof(lowat));
}
enum sp_egress_observation_kind {
SP_EGRESS_OBSERVATION_NONE = 0,
SP_EGRESS_OBSERVATION_CREDIT = 1,
SP_EGRESS_OBSERVATION_EOF = 2,
};
SP_TEST_API static int sp_egress_observe_backend_typed(
struct sp_egress_executor_flow *flow, int backend_fd, uint64_t now_ns,
int read_half_closed, int force_fallback, uint64_t global_flight_credit,
uint64_t effect_byte_credit, uint8_t *discard, size_t discard_capacity,
uint8_t *advisory, size_t advisory_capacity, struct sp_ev_credit *credit,
struct sp_ev_stream_eof *eof) {
uint64_t represented;
uint64_t per_flow_credit;
uint64_t cap;
if (flow == NULL || credit == NULL || eof == NULL ||
flow->stream_tail < flow->stream_consumed)
return -1;
represented = flow->stream_tail - flow->stream_consumed;
if (represented > SP_TCP_FLIGHT_PER_FLOW)
return -1;
per_flow_credit = SP_TCP_FLIGHT_PER_FLOW - represented;
cap = sp_egress_min_u64(per_flow_credit, global_flight_credit);
cap = sp_egress_min_u64(cap, effect_byte_credit);
cap = sp_egress_min_u64(cap,
(uint64_t)SP_TCP_CORE_MAX_SEGMENTS *
SP_TCP_CORE_MAX_MSS);
cap = sp_egress_min_u64(cap, advisory_capacity);
if (cap == 0)
return 0;
struct sp_egress_peek_result peek = sp_egress_peek_backend(
backend_fd, flow->stream_consumed, flow->stream_tail, read_half_closed,
force_fallback, discard, discard_capacity, advisory, (size_t)cap);
if (peek.status == SP_EGRESS_IO_BLOCKED) {
if (sp_egress_arm_backend_lowat(backend_fd, flow) < 0)
return -1;
return 0;
}
if (peek.status == SP_EGRESS_IO_FATAL)
return -1;
if (peek.status == SP_EGRESS_IO_EOF) {
if (flow->eof_reported)
return 0;
flow->eof_reported = 1;
memset(eof, 0, sizeof(*eof));
sp_tcp_event_context_init(&eof->context, flow->flow, flow->generation,
now_ns);
eof->stream_tail.value = flow->stream_tail;
return SP_EGRESS_OBSERVATION_EOF;
}
if (peek.observed > cap || UINT64_MAX - flow->stream_tail < peek.observed)
return -1;
struct sp_egress_executor_flow advanced = *flow;
advanced.stream_tail += peek.observed;
if (sp_egress_arm_backend_lowat(backend_fd, &advanced) < 0)
return -1;
flow->stream_tail = advanced.stream_tail;
memset(credit, 0, sizeof(*credit));
sp_tcp_event_context_init(&credit->context, flow->flow, flow->generation,
now_ns);
credit->stream_tail.value = flow->stream_tail;
return SP_EGRESS_OBSERVATION_CREDIT;
}
SP_TEST_API static int sp_egress_materialize_emit(
struct sp_egress_executor_flow *flow, int backend_fd,
const struct sp_fx_tcp_emit_batch *effect, int force_fallback,
uint8_t *discard,
size_t discard_capacity, uint8_t *payload, size_t payload_capacity) {
uint64_t length;
if (flow == NULL || effect == NULL ||
effect->context.key.flow.value != flow->flow ||
effect->context.key.generation.value != flow->generation ||
effect->end.value < effect->start.value ||
effect->start.value < flow->stream_consumed ||
effect->end.value > flow->stream_tail)
return -1;
length = effect->end.value - effect->start.value;
if (length == 0 || length > payload_capacity ||
length > (uint64_t)SP_TCP_CORE_MAX_SEGMENTS * SP_TCP_CORE_MAX_MSS)
return -1;
struct sp_egress_peek_result peek = sp_egress_peek_backend(
backend_fd, flow->stream_consumed, effect->start.value, 0, force_fallback,
discard, discard_capacity, payload, (size_t)length);
if (peek.status == SP_EGRESS_IO_BLOCKED)
return 0;
if (peek.status != SP_EGRESS_IO_PROGRESS || peek.observed != length) {
errno = EPROTO;
return -1;
}
return 1;
}
SP_TEST_API static int sp_egress_execute_consume_typed(
struct sp_egress_executor_flow *flow, int backend_fd,
const struct sp_fx_stream_consume *effect, uint64_t now_ns,
int force_scratch,
uint8_t *scratch, size_t scratch_capacity,
struct sp_ev_stream_consumed *result) {
uint64_t authorized;
if (flow == NULL || effect == NULL || result == NULL ||
effect->context.key.flow.value != flow->flow ||
effect->context.key.generation.value != flow->generation ||
effect->end.value <= effect->start.value ||
effect->start.value != flow->stream_consumed)
return -1;
authorized = effect->end.value - effect->start.value;
if (authorized > SIZE_MAX)
return -1;
struct sp_egress_consume_result consume = sp_egress_consume_backend(
backend_fd, (size_t)authorized, force_scratch, scratch, scratch_capacity);
if (consume.status == SP_EGRESS_IO_BLOCKED)
return 0;
if (consume.status == SP_EGRESS_IO_FATAL)
return -1;
memset(result, 0, sizeof(*result));
sp_tcp_event_context_init(&result->context, flow->flow, flow->generation,
now_ns);
result->effect = effect->context.effect;
if (consume.status == SP_EGRESS_IO_EOF) {
return 1;
}
if (consume.consumed == 0 || consume.consumed > authorized ||
UINT64_MAX - flow->stream_consumed < consume.consumed)
return -1;
flow->stream_consumed += consume.consumed;
if (sp_egress_arm_backend_lowat(backend_fd, flow) < 0)
return -1;
result->accepted.value = consume.consumed;
return 1;
}
SP_TEST_API static int sp_egress_execute_emit_with_typed(
struct sp_egress_executor_flow *flow, sp_egress_write_fn write_fn, void *context,
const struct sp_fx_tcp_emit_batch *effect, uint32_t src, uint32_t dst,
uint16_t src_port, uint16_t dst_port, const uint8_t *payload,
size_t payload_len, int force_software, uint64_t now_ns,
struct sp_ev_tun_emit_result *result) {
size_t logical_segments;
size_t emitted_records = 0;
size_t accepted_segments = 0;
uint64_t accepted = 0;
if (flow == NULL || effect == NULL || result == NULL ||
effect->context.key.flow.value != flow->flow ||
effect->context.key.generation.value != flow->generation ||
flow->next_control_seq == UINT64_MAX)
return -1;
if (force_software) {
struct sp_tcp_emit_cursor cursor;
struct sp_tcp_emit_cursor next_cursor;
uint8_t scalar_record[SP_TUN_CONTROL_FRAME_MAX];
if (sp_tcp_emit_cursor_init(effect, payload_len, &cursor) < 0)
return -1;
logical_segments = cursor.segment_count;
for (;;) {
size_t segment_payload_len;
next_cursor = cursor;
size_t scalar_len = sp_tcp_emit_format_scalar(
effect, src, dst, src_port, dst_port, payload, payload_len,
&next_cursor,
scalar_record, sizeof(scalar_record), &segment_payload_len);
if (scalar_len == 0 || segment_payload_len == 0)
return -1;
enum sp_egress_io_status status = sp_egress_write_tun_record_with(
write_fn, context, scalar_record, scalar_len);
if (status == SP_EGRESS_IO_BLOCKED) {
if (accepted == 0)
return 0;
break;
}
if (status != SP_EGRESS_IO_PROGRESS)
return -1;
cursor = next_cursor;
accepted += segment_payload_len;
emitted_records++;
accepted_segments++;
if (cursor.segment_index == cursor.segment_count)
break;
}
} else {
uint8_t record[SP_TUN_FRAME_MAX_BYTES];
size_t record_len;
record_len = sp_egress_build_tcp_emit_record(
record, sizeof(record), src, dst, src_port, dst_port, effect, payload,
payload_len, &logical_segments);
if (record_len == 0)
return -1;
enum sp_egress_io_status status =
sp_egress_write_tun_record_with(write_fn, context, record, record_len);
if (status == SP_EGRESS_IO_BLOCKED)
return 0;
if (status != SP_EGRESS_IO_PROGRESS)
return -1;
accepted = payload_len;
emitted_records = 1;
accepted_segments = logical_segments;
}
if (accepted == 0 || accepted > payload_len)
return -1;
sp_counter_add(&g_resource_counters.tcp_logical_segments_host_to_child,
accepted_segments);
sp_counter_add(&g_resource_counters.tcp_payload_bytes_host_to_child,
accepted);
if (!force_software && logical_segments > 1) {
sp_counter_inc(&g_resource_counters.tcp_gso_frames_host_to_child);
sp_counter_add(&g_resource_counters.tcp_gso_segments_host_to_child,
accepted_segments);
} else {
sp_counter_add(&g_resource_counters.tun_vnet_scalar_egress,
emitted_records);
}
memset(result, 0, sizeof(*result));
sp_tcp_event_context_init(&result->context,
effect->context.key.flow.value,
effect->context.key.generation.value, now_ns);
result->context.control_sequence.value = ++flow->next_control_seq;
result->effect = effect->context.effect;
result->accepted.value = accepted;
result->outcome = SP_EMIT_ACCEPTED;
return 1;
}
/* Child-to-backend K_rx staging, durable reservation, and STREAM_SEND
* execution. The pure core authorizes an exact ingress slice against the
* optimistic frontier Q (rx_authorized_end_abs); this executor stages up to
* K_rx immutable frames per flow, submits the exact authorized slice to the
* backend stream socket, and reports the positive backend-accepted prefix
* (advancing A) and the durably-retained suffix (advancing P) back to the core
* as an sp_ev_stream_send_result. Frame bytes never enter the engine. A short
* write copies the unaccepted suffix into the per-flow durable reservation
* before releasing transient staging; EAGAIN copies the complete slice; the
* core re-emits STREAM_SEND with capability 0 to re-flush the reservation once
* the socket is writable. */
#define SP_INGRESS_K_RX_COMPUTED \
((SP_TCP_PENDING_PER_FLOW + \
(SP_TCP_GSO_POLICY_SEGS * SP_TCP_MTU1500_MAX_MSS) - 1) / \
(SP_TCP_GSO_POLICY_SEGS * SP_TCP_MTU1500_MAX_MSS))
enum {
SP_INGRESS_K_RX = SP_INGRESS_K_RX_COMPUTED,
};
_Static_assert(SP_INGRESS_K_RX == 6, "A5 per-flow K_rx geometry changed");
_Static_assert(SP_INGRESS_STAGE_SLOTS >= 2 * SP_INGRESS_K_RX,
"global staging slots must cover 2*K_rx");
enum sp_ingress_stage_state {
SP_INGRESS_STAGE_FREE = 0,
SP_INGRESS_STAGE_STAGED,
};
struct sp_ingress_stage_slot {
enum sp_ingress_stage_state state;
uint64_t flow;
uint64_t generation;
uint64_t ingress_id;
uint64_t interval_start;
uint64_t interval_end;
uint8_t *payload;
};
struct sp_ingress_stage_pool {
struct sp_ingress_stage_slot slots[SP_INGRESS_STAGE_SLOTS];
uint8_t *arena;
size_t global_reserved;
uint64_t next_ingress_id;
};
struct sp_ingress_flow {
uint64_t flow;
uint64_t generation;
uint64_t reserved_base;
size_t pending_bytes;
unsigned staged;
uint16_t reservation_slot;
};
SP_TEST_API static int sp_ingress_stage_pool_init(struct sp_ingress_stage_pool *pool,
uint8_t *arena,
size_t arena_capacity) {
if (pool == NULL || arena == NULL ||
arena_capacity <
(size_t)SP_INGRESS_STAGE_SLOTS * (size_t)SP_TUN_FRAME_MAX_BYTES)
return -1;
memset(pool, 0, sizeof(*pool));
pool->arena = arena;
return 0;
}
SP_TEST_API static void sp_ingress_flow_init(struct sp_ingress_flow *flow,
uint64_t flow_id,
uint64_t generation) {
memset(flow, 0, sizeof(*flow));
flow->flow = flow_id;
flow->generation = generation;
flow->reservation_slot = UINT16_MAX;
}
static struct sp_ingress_stage_slot *sp_ingress_find_slot(struct sp_ingress_stage_pool *pool,
uint64_t flow,
uint64_t generation,
uint64_t ingress_id) {
if (ingress_id == 0)
return NULL;
for (size_t i = 0; i < SP_INGRESS_STAGE_SLOTS; ++i) {
struct sp_ingress_stage_slot *slot = &pool->slots[i];
if (slot->state != SP_INGRESS_STAGE_FREE && slot->ingress_id == ingress_id &&
slot->flow == flow && slot->generation == generation)
return slot;
}
return NULL;
}
static void sp_ingress_stage_free(struct sp_ingress_flow *flow,
struct sp_ingress_stage_slot *slot) {
if (slot->state == SP_INGRESS_STAGE_FREE)
return;
if (flow != NULL && flow->staged > 0)
flow->staged--;
memset(slot, 0, sizeof(*slot));
slot->state = SP_INGRESS_STAGE_FREE;
}
/* Admit one immutable child frame. Returns 1 with *ingress_id_out set, 0 for a
* typed STAGE_BLOCKED (per-flow K_rx cap or global slot exhaustion), or -1 on
* invalid input. */
static int sp_ingress_stage_store(
struct sp_ingress_stage_pool *pool, struct sp_ingress_flow *flow,
uint64_t interval_start, const uint8_t *payload, size_t payload_len,
uint64_t *ingress_id_out) {
size_t free_index = SP_INGRESS_STAGE_SLOTS;
struct sp_ingress_stage_slot *slot;
if (pool == NULL || flow == NULL || ingress_id_out == NULL ||
payload == NULL || payload_len == 0 ||
payload_len > SP_TUN_FRAME_MAX_BYTES ||
UINT64_MAX - interval_start < payload_len ||
pool->next_ingress_id == UINT64_MAX)
return -1;
for (size_t i = 0; i < SP_INGRESS_STAGE_SLOTS; ++i) {
if (pool->slots[i].state == SP_INGRESS_STAGE_FREE) {
free_index = i;
break;
}
}
if (free_index == SP_INGRESS_STAGE_SLOTS)
return 0;
slot = &pool->slots[free_index];
slot->payload = pool->arena + free_index * (size_t)SP_TUN_FRAME_MAX_BYTES;
memcpy(slot->payload, payload, payload_len);
slot->state = SP_INGRESS_STAGE_STAGED;
slot->flow = flow->flow;
slot->generation = flow->generation;
/* IDs are a per-pool sequence: never zero and never reused, so a stale
* completion cannot match a slot that was freed and staged again. */
slot->ingress_id = ++pool->next_ingress_id;
slot->interval_start = interval_start;
slot->interval_end = interval_start + payload_len;
flow->staged++;
*ingress_id_out = slot->ingress_id;
return 1;
}
SP_TEST_API static int sp_ingress_stage_admit(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow,
uint64_t interval_start,
const uint8_t *payload,
size_t payload_len,
uint64_t *ingress_id_out) {
if (flow->staged >= SP_INGRESS_K_RX)
return 0;
return sp_ingress_stage_store(pool, flow, interval_start, payload,
payload_len, ingress_id_out);
}
/* Dispose every live capability for a flow in finite bounded work. */
SP_TEST_API static size_t sp_ingress_stage_cancel_flow(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow) {
size_t disposed = 0;
for (size_t i = 0; i < SP_INGRESS_STAGE_SLOTS; ++i) {
struct sp_ingress_stage_slot *slot = &pool->slots[i];
if (slot->state != SP_INGRESS_STAGE_FREE && slot->flow == flow->flow &&
slot->generation == flow->generation) {
sp_ingress_stage_free(flow, slot);
disposed++;
}
}
return disposed;
}
static int sp_ingress_reservation_capacity(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow,
size_t len,
size_t reservation_capacity) {
if (flow->pending_bytes > reservation_capacity ||
flow->pending_bytes > (size_t)SP_TCP_PENDING_PER_FLOW ||
pool->global_reserved > (size_t)SP_TCP_PENDING_ALL ||
flow->pending_bytes > pool->global_reserved ||
(flow->pending_bytes != 0 &&
flow->reserved_base > UINT64_MAX - flow->pending_bytes))
return -1;
if (len > reservation_capacity - flow->pending_bytes ||
len > (size_t)SP_TCP_PENDING_PER_FLOW - flow->pending_bytes ||
len > (size_t)SP_TCP_PENDING_ALL - pool->global_reserved) {
return 0;
}
return 1;
}
/* Append durably-retained bytes to the per-flow reservation head. Returns 1 on
* success, 0 for STAGE_BLOCKED (byte cap or buffer exhaustion, counted), -1 on
* an invalid reservation invariant or non-contiguous append. */
static int sp_ingress_reserve_append(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow,
uint64_t abs_start, const uint8_t *bytes,
size_t len, uint8_t *reservation,
size_t reservation_capacity) {
if (len == 0)
return 1;
if (flow->pending_bytes != 0 &&
(flow->reserved_base > UINT64_MAX - flow->pending_bytes ||
abs_start != flow->reserved_base + flow->pending_bytes))
return -1;
{
int capacity = sp_ingress_reservation_capacity(
pool, flow, len, reservation_capacity);
if (capacity <= 0)
return capacity;
}
if (flow->pending_bytes == 0)
flow->reserved_base = abs_start;
memcpy(reservation + flow->pending_bytes, bytes, len);
flow->pending_bytes += len;
pool->global_reserved += len;
return 1;
}
/* Retire the positive backend-accepted prefix from the reservation head. */
static void sp_ingress_reserve_consume(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow,
uint8_t *reservation, size_t len) {
if (len == 0 || len > flow->pending_bytes || len > pool->global_reserved)
return;
memmove(reservation, reservation + len, flow->pending_bytes - len);
flow->pending_bytes -= len;
flow->reserved_base += len;
pool->global_reserved -= len;
}
typedef sp_tcp_write_fn sp_ingress_send_fn;
/* Classify one nonblocking backend send through the shared write adapter. On
* PROGRESS *accepted holds the positive prefix (full or partial); BLOCKED is
* EAGAIN with nothing accepted; FATAL includes zero and over-reported writes. */
static enum sp_egress_io_status
sp_ingress_send_slice_with(sp_ingress_send_fn send_fn, void *context, const uint8_t *buf,
size_t len, size_t *accepted) {
return sp_tcp_write_slice_with(send_fn, context, buf, len, 0, accepted);
}
SP_TEST_API static ssize_t sp_ingress_kernel_send(void *context, const uint8_t *buf,
size_t len) {
int fd = (int)(intptr_t)context;
return send(fd, buf, len, MSG_DONTWAIT | MSG_NOSIGNAL);
}
/* Execute one SP_FX_STREAM_SEND effect. A nonzero capability authorizes a fresh
* staged frame; a zero capability re-flushes the durable reservation. Returns
* 1 with an sp_ev_stream_send_result (accepted/retained), 0 for a typed
* STAGE_BLOCKED that leaves the effect live for retry, or -1 flow-fatal. */
SP_TEST_API static int sp_ingress_execute_stream_send_typed(
struct sp_ingress_stage_pool *pool, struct sp_ingress_flow *flow,
sp_ingress_send_fn send_fn, void *context,
const struct sp_fx_stream_send *effect,
uint64_t now_ns, uint8_t *reservation, size_t reservation_capacity,
struct sp_ev_stream_send_result *result) {
struct sp_ingress_stage_slot *slot = NULL;
const uint8_t *src;
uint64_t len64;
size_t len;
size_t accepted = 0;
size_t retained;
int queued_behind_reservation = 0;
enum sp_egress_io_status status;
if (pool == NULL || flow == NULL || effect == NULL || result == NULL ||
reservation == NULL || effect->context.key.flow.value != flow->flow ||
effect->context.key.generation.value != flow->generation ||
effect->end.value <= effect->start.value)
return -1;
len64 = effect->end.value - effect->start.value;
if (len64 > SIZE_MAX || len64 > reservation_capacity)
return -1;
len = (size_t)len64;
/* Both fresh sends and capability-zero reflushes rely on the retained-byte
* counters before they can touch reservation storage. */
if (sp_ingress_reservation_capacity(pool, flow, 0,
reservation_capacity) != 1)
return -1;
if (effect->capability.value != 0) {
slot = sp_ingress_find_slot(pool, flow->flow, flow->generation,
effect->capability.value);
if (slot == NULL || effect->start.value < slot->interval_start ||
effect->end.value > slot->interval_end)
return -1;
src = slot->payload +
(size_t)(effect->start.value - slot->interval_start);
/* Reserve the exact slice against both byte caps before the first send. */
{
int capacity = sp_ingress_reservation_capacity(
pool, flow, len, reservation_capacity);
if (capacity <= 0)
return capacity;
}
} else {
if (flow->pending_bytes == 0 ||
effect->start.value != flow->reserved_base ||
len > flow->pending_bytes)
return -1;
src = reservation;
}
/* A durable prefix is already waiting at [A,P). A later fresh capability
* starts at P, so sending it directly would put bytes after the gap on the
* backend socket and then report a non-head accepted prefix to the core.
* Preserve wire order mechanically: append the whole fresh slice to the
* existing reservation and report it retained without issuing send(2).
* EPOLLOUT later re-flushes the one contiguous [A,P) reservation. */
if (effect->capability.value != 0 && flow->pending_bytes != 0) {
int reserved = sp_ingress_reserve_append(
pool, flow, effect->start.value, src, len, reservation,
reservation_capacity);
if (reserved <= 0)
return reserved;
retained = len;
queued_behind_reservation = 1;
} else {
status = sp_ingress_send_slice_with(send_fn, context, src, len, &accepted);
if (status == SP_EGRESS_IO_FATAL || accepted > len)
return -1;
retained = len - accepted;
}
if (effect->capability.value != 0) {
if (retained > 0 && !queued_behind_reservation) {
int reserved =
sp_ingress_reserve_append(pool, flow, effect->start.value + accepted,
src + accepted, retained, reservation,
reservation_capacity);
if (reserved <= 0)
return reserved;
}
sp_ingress_stage_free(flow, slot);
} else if (accepted > 0) {
sp_ingress_reserve_consume(pool, flow, reservation, accepted);
}
memset(result, 0, sizeof(*result));
sp_tcp_event_context_init(&result->context, flow->flow, flow->generation,
now_ns);
result->effect = effect->context.effect;
result->accepted.value = accepted;
result->retained.value = retained;
result->outcome = retained != 0 ? SP_SEND_RETRY : SP_SEND_PROGRESS;
return 1;
}
/* Release a staged capability on the core's SP_FX_INGRESS_DISPOSE and
* acknowledge with an sp_ev_ingress_disposed. Idempotent: a capability already
* released by a completed STREAM_SEND still acknowledges. */
SP_TEST_API static int sp_ingress_execute_dispose_typed(struct sp_ingress_stage_pool *pool,
struct sp_ingress_flow *flow,
const struct sp_fx_ingress_dispose *effect,
uint64_t now_ns,
struct sp_ev_ingress_disposed *result) {
struct sp_ingress_stage_slot *slot;
if (pool == NULL || flow == NULL || effect == NULL || result == NULL ||
effect->context.key.flow.value != flow->flow ||
effect->context.key.generation.value != flow->generation)
return -1;
slot = sp_ingress_find_slot(pool, flow->flow, flow->generation,
effect->ingress.value);
if (slot != NULL) {
sp_ingress_stage_free(flow, slot);
}
memset(result, 0, sizeof(*result));
sp_tcp_event_context_init(&result->context, flow->flow, flow->generation,
now_ns);
result->effect = effect->context.effect;
return 1;
}
/* TUN egress scheduler: dual control/data FIFOs, per-flow
* byte deficit round robin, and credit epochs.
*
* This is upstream of, not a replacement for, `g_tun_queue`. The scheduler
* owns typed flow/class/cost records before emission. `tun_write_frame()` may
* then retain an already-scheduled opaque frame in the final kernel-EAGAIN
* FIFO; scalar UDP/publish producers also use that final FIFO. Collapsing the
* two requires one common final-emission ownership contract, not merely a size
* saving. One opaque FIFO alone cannot implement class bypass or byte DRR.
*
* Records are admitted with an explicit class and cost, and serviced against an
* injected write sink so the whole package is testable without a TUN fd. */
enum sp_scheduler_class {
SP_SCHEDULER_CLASS_DATA = 0,
SP_SCHEDULER_CLASS_CONTROL,
};
/* Control records carry a reserve tier. Terminal SYN/FIN/RST effects and
* ordinary response ACKs have named slot reserves that ordinary control traffic
* may not consume. */
enum sp_scheduler_ctrl_tier {
SP_SCHEDULER_CTRL_ORDINARY = 0,
SP_SCHEDULER_CTRL_RESPONSE,
SP_SCHEDULER_CTRL_TERMINAL,
};
/* Dimensions a producer can be blocked on. A blocked producer records every
* responsible dimension and does not retry until a greater credit epoch reports
* growth in one of them. */
enum sp_scheduler_credit_dim {
SP_SCHEDULER_DIM_DATA_SLOTS = 1U << 0,
SP_SCHEDULER_DIM_DATA_BYTES = 1U << 1,
SP_SCHEDULER_DIM_CONTROL_SLOTS = 1U << 2,
SP_SCHEDULER_DIM_CONTROL_BYTES = 1U << 3,
};
enum {
/* DRR metadata must cover every concurrently admissible outbound TCP owner.
* This is a small control structure, not per-flow payload storage. */
SP_SCHEDULER_MAX_FLOWS = MAX_TCP,
/* Per-logical-packet cost charged on top of bytes, so a flow sending many
* tiny packets is not serviced as cheaply as its byte count suggests. Never
* charged per GSO container: a 32-segment frame is charged as 32 logical
* packets, not as one. */
SP_SCHEDULER_PACKET_COST = 64,
/* One DRR round grants this many bytes of deficit to each active flow. */
SP_SCHEDULER_DRR_QUANTUM = 16 * 1024,
/* Carry is capped at the charge of one maximum authorized batch, so an idle
* flow cannot accumulate unbounded credit and then monopolise a round. This
* must account for the per-logical-packet cost of every segment in that
* batch, not one: a cap below the maximum charge would make the largest
* legal record permanently unaffordable. */
SP_SCHEDULER_DRR_CARRY_MAX = SP_TCP_GSO_POLICY_SEGS *
(SP_TCP_MTU1500_MAX_MSS + SP_SCHEDULER_PACKET_COST),
/* A flow inactive for this many rounds has its deficit reset. */
SP_SCHEDULER_DRR_IDLE_ROUNDS = 4,
/* Hard bound on round advances per service call. The clock is supplied by
* the caller and a caller may legitimately hold it still, so termination may
* not depend on time passing. */
SP_SCHEDULER_ROUNDS_PER_CALL = 64,
/* Per-slot arena stride. Resource geometry sizes the data arena as 32 x
* 64 KiB, which is
* nine bytes short of 32 full SP_TUN_FRAME_MAX_BYTES frames, so the data lane
* admits frames up to the stride rather than the absolute frame ceiling.
* That is not a practical restriction: the largest authorized batch is 32
* segments x 1460 bytes plus headers, far below the stride. */
SP_SCHEDULER_CONTROL_STRIDE = SP_TUN_CONTROL_FRAME_MAX,
SP_SCHEDULER_DATA_STRIDE = SP_TUN_DATA_MAX_BYTES / SP_TUN_DATA_MAX_PKTS,
};
_Static_assert(SP_SCHEDULER_CONTROL_STRIDE * SP_TUN_CONTROL_MAX_PKTS <=
SP_TUN_CONTROL_MAX_BYTES,
"scheduler control slots must fit the resource control arena");
_Static_assert(SP_SCHEDULER_DATA_STRIDE * SP_TUN_DATA_MAX_PKTS <=
SP_TUN_DATA_MAX_BYTES,
"scheduler data slots must fit the resource data arena");
_Static_assert(SP_SCHEDULER_DATA_STRIDE > SP_VNET_HDR_BYTES + SP_IPV4_HEADER_BYTES +
SP_TCP_GSO_POLICY_SEGS *
SP_TCP_MTU1500_MAX_MSS,
"scheduler data stride must hold a maximum authorized GSO batch");
/* The largest admissible record must be affordable, or DRR could never issue
* it and the service loop would spin advancing rounds forever. */
_Static_assert(SP_SCHEDULER_DRR_CARRY_MAX >= SP_TCP_GSO_POLICY_SEGS *
SP_TCP_MTU1500_MAX_MSS +
SP_TCP_GSO_POLICY_SEGS *
SP_SCHEDULER_PACKET_COST,
"scheduler carry cap must cover the maximum authorized batch charge");
struct sp_scheduler_record {
uint64_t flow;
uint64_t generation;
/* UINT16_MAX means that the record has no live executor owner (for example
* an auxiliary CLOSED response). Live producers carry the checked TCP
* table index so the final sink never reverses an identity through a table
* scan. */
uint16_t flow_index;
enum sp_scheduler_class klass;
enum sp_scheduler_ctrl_tier tier;
/* Absolute sequence this record occupies, and for control records the
* sequence that must already be committed before it may be serviced. */
uint64_t seq_abs;
uint64_t depends_on_seq;
uint64_t logical_bytes;
unsigned logical_packets;
uint8_t terminal;
size_t frame_len;
size_t frame_offset;
uint8_t in_use;
};
/* Each slot owns a fixed stride of the arena, so a frame never moves and
* out-of-order removal cannot fragment the arena. FIFO order is kept in a
* separate small array of slot indices; removing a record shifts those indices
* rather than any frame bytes. */
struct sp_scheduler_queue {
struct sp_scheduler_record records[SP_TUN_CONTROL_MAX_PKTS];
uint16_t order[SP_TUN_CONTROL_MAX_PKTS];
size_t capacity_slots;
size_t capacity_bytes;
size_t stride;
size_t count;
size_t bytes;
uint8_t *arena;
};
struct sp_scheduler_flow {
uint64_t flow;
int64_t deficit;
uint64_t committed_seq;
uint64_t last_round;
size_t queued_records;
uint8_t active;
};
struct sp_scheduler_credit {
uint64_t epoch;
size_t data_slots;
size_t data_bytes;
size_t control_slots;
size_t control_bytes;
};
struct sp_scheduler_sched {
struct sp_scheduler_queue control;
struct sp_scheduler_queue data;
struct sp_scheduler_flow flows[SP_SCHEDULER_MAX_FLOWS];
size_t flow_count;
uint64_t round;
uint64_t epoch;
};
_Static_assert(MAX_TCP < UINT16_MAX,
"scheduler flow index sentinel must not alias a TCP slot");
struct sp_scheduler_turn {
size_t packets;
size_t bytes;
size_t records;
uint64_t started_ns;
uint8_t write_blocked;
};
/* Sink verdict for one offered record. FAILED stops service for broker cleanup;
* RETIRED removes the record without counting or charging its flow's deficit. */
enum {
SP_SCHEDULER_WRITE_FAILED = -2,
SP_SCHEDULER_WRITE_BLOCKED = -1,
SP_SCHEDULER_WRITE_DONE = 0,
SP_SCHEDULER_WRITE_RETIRED = 1,
};
/* The record and its frame are valid only for the call; the sink must not
* retain them or mutate the scheduler. */
typedef int (*sp_scheduler_write_fn)(void *context,
const struct sp_scheduler_record *record,
const uint8_t *frame);
/* Supplies the current monotonic time. Passing NULL disables the nanosecond
* budgets and leaves the count budgets and the round bound as the termination
* guarantee. */
typedef uint64_t (*sp_scheduler_clock_fn)(void *context);
static void sp_scheduler_queue_init(struct sp_scheduler_queue *queue, uint8_t *arena,
size_t capacity_slots, size_t capacity_bytes,
size_t stride) {
queue->arena = arena;
queue->capacity_slots = capacity_slots;
queue->capacity_bytes = capacity_bytes;
queue->stride = stride;
}
static size_t sp_scheduler_queue_free_slot(const struct sp_scheduler_queue *queue) {
size_t i;
for (i = 0; i < queue->capacity_slots; ++i)
if (!queue->records[i].in_use)
return i;
return SIZE_MAX;
}
/* `sched` must be zero-filled: the broker's static instance is, and clearing
* it again would fault in its 56 KB just to write zeros (callers that reuse
* storage, such as tests, clear it themselves). */
SP_TEST_API static void sp_scheduler_init(struct sp_scheduler_sched *sched,
uint8_t *control_arena,
size_t control_arena_len,
uint8_t *data_arena,
size_t data_arena_len) {
(void)control_arena_len;
(void)data_arena_len;
sp_scheduler_queue_init(&sched->control, control_arena, SP_TUN_CONTROL_MAX_PKTS,
SP_TUN_CONTROL_MAX_BYTES, SP_SCHEDULER_CONTROL_STRIDE);
sp_scheduler_queue_init(&sched->data, data_arena, SP_TUN_DATA_MAX_PKTS,
SP_TUN_DATA_MAX_BYTES, SP_SCHEDULER_DATA_STRIDE);
sched->epoch = 1;
}
static struct sp_scheduler_flow *sp_scheduler_flow_lookup(struct sp_scheduler_sched *sched,
uint64_t flow, int create) {
size_t i;
for (i = 0; i < sched->flow_count; ++i)
if (sched->flows[i].flow == flow)
return &sched->flows[i];
if (!create || sched->flow_count == SP_SCHEDULER_MAX_FLOWS)
return NULL;
sched->flows[sched->flow_count].flow = flow;
sched->flows[sched->flow_count].deficit = 0;
sched->flows[sched->flow_count].committed_seq = 0;
sched->flows[sched->flow_count].last_round = sched->round;
sched->flows[sched->flow_count].queued_records = 0;
sched->flows[sched->flow_count].active = 0;
return &sched->flows[sched->flow_count++];
}
/* Charge is bytes plus a fixed cost per *logical* packet. A GSO container is
* charged for every segment it expands into, so a 64 KiB super-frame cannot
* receive the same service as a 64-byte control packet. */
static uint64_t sp_scheduler_charge(const struct sp_scheduler_record *record) {
unsigned packets = record->logical_packets == 0 ? 1 : record->logical_packets;
return record->logical_bytes + (uint64_t)packets * SP_SCHEDULER_PACKET_COST;
}
/* Slots a class must leave free for its reserve tiers. */
static size_t sp_scheduler_control_reserve_for(enum sp_scheduler_ctrl_tier tier) {
switch (tier) {
case SP_SCHEDULER_CTRL_TERMINAL:
return 0;
case SP_SCHEDULER_CTRL_RESPONSE:
return SP_TUN_TERMINAL_RESERVE;
case SP_SCHEDULER_CTRL_ORDINARY:
default:
return (size_t)SP_TUN_TERMINAL_RESERVE + SP_TUN_RESPONSE_RESERVE;
}
}
static void sp_scheduler_publish_epoch(struct sp_scheduler_sched *sched) {
sched->epoch++;
}
SP_TEST_API static struct sp_scheduler_credit sp_scheduler_credit_now(
const struct sp_scheduler_sched *sched) {
struct sp_scheduler_credit credit;
credit.epoch = sched->epoch;
credit.data_slots = sched->data.capacity_slots - sched->data.count;
credit.data_bytes = sched->data.capacity_bytes - sched->data.bytes;
credit.control_slots = sched->control.capacity_slots - sched->control.count;
credit.control_bytes = sched->control.capacity_bytes - sched->control.bytes;
return credit;
}
/* True when `later` reports growth, versus `earlier`, in at least one of the
* dimensions the producer was blocked on. Readiness alone is not progress: a
* greater epoch with no growth in a blocked dimension must not trigger a retry. */
SP_TEST_API static int sp_scheduler_credit_permits_retry(
const struct sp_scheduler_credit *earlier, const struct sp_scheduler_credit *later,
unsigned blocked_dims) {
if (later->epoch <= earlier->epoch)
return 0;
if ((blocked_dims & SP_SCHEDULER_DIM_DATA_SLOTS) != 0 &&
later->data_slots > earlier->data_slots)
return 1;
if ((blocked_dims & SP_SCHEDULER_DIM_DATA_BYTES) != 0 &&
later->data_bytes > earlier->data_bytes)
return 1;
if ((blocked_dims & SP_SCHEDULER_DIM_CONTROL_SLOTS) != 0 &&
later->control_slots > earlier->control_slots)
return 1;
if ((blocked_dims & SP_SCHEDULER_DIM_CONTROL_BYTES) != 0 &&
later->control_bytes > earlier->control_bytes)
return 1;
return 0;
}
/* Admits one framed record. Returns 0 on success; -1 with *blocked_dims set to
* every responsible dimension when the record does not fit. Data may consume
* neither control slots nor control bytes, and never falsely reports a commit. */
SP_TEST_API static int sp_scheduler_admit_with_flow_index(
struct sp_scheduler_sched *sched, const struct sp_scheduler_record *record,
const uint8_t *frame, size_t frame_len, unsigned *blocked_dims,
uint16_t flow_index) {
struct sp_scheduler_queue *queue;
struct sp_scheduler_flow *flow = NULL;
struct sp_scheduler_record *slot;
size_t reserve;
size_t tail;
unsigned blocked = 0;
*blocked_dims = 0;
if (flow_index != UINT16_MAX &&
((size_t)flow_index >= sp_ns_tcp_slot_limit() || flow_index >= MAX_TCP))
return -1;
if (frame_len == 0 || record->generation == 0)
return -1;
/* A record whose charge exceeds the carry cap could never be issued by DRR.
* Refusing it at admission keeps "every admitted record is eventually
* affordable" an enforced invariant rather than an assumption. */
if (record->klass == SP_SCHEDULER_CLASS_DATA &&
sp_scheduler_charge(record) > (uint64_t)SP_SCHEDULER_DRR_CARRY_MAX)
return -1;
if (record->klass == SP_SCHEDULER_CLASS_CONTROL) {
queue = &sched->control;
/* Control/response records are scalar and never use GSO, so the byte proof
* cannot be bypassed by a super-frame on the control lane. */
if (frame_len > SP_TUN_CONTROL_FRAME_MAX)
return -1;
reserve = sp_scheduler_control_reserve_for(record->tier);
if (queue->count + reserve >= queue->capacity_slots &&
!(record->tier == SP_SCHEDULER_CTRL_TERMINAL &&
queue->count < queue->capacity_slots))
blocked |= SP_SCHEDULER_DIM_CONTROL_SLOTS;
if (queue->bytes + frame_len > queue->capacity_bytes)
blocked |= SP_SCHEDULER_DIM_CONTROL_BYTES;
} else {
queue = &sched->data;
if (queue->count >= queue->capacity_slots)
blocked |= SP_SCHEDULER_DIM_DATA_SLOTS;
if (queue->bytes + frame_len > queue->capacity_bytes)
blocked |= SP_SCHEDULER_DIM_DATA_BYTES;
}
if (frame_len > queue->stride)
return -1;
if (blocked != 0) {
*blocked_dims = blocked;
return -1;
}
/* Control-only/auxiliary responses need no DRR or committed-data state.
* Allocating a flow slot for every CLOSED RST would let historical hostile
* tuples exhaust metadata needed by live data owners. */
if (record->klass == SP_SCHEDULER_CLASS_DATA) {
flow = sp_scheduler_flow_lookup(sched, record->flow, 1);
if (flow == NULL) {
*blocked_dims = SP_SCHEDULER_DIM_DATA_SLOTS;
return -1;
}
} else if (flow_index != UINT16_MAX) {
flow = sp_scheduler_flow_lookup(sched, record->flow, 0);
}
if (flow != NULL && flow->queued_records == SIZE_MAX)
return -1;
tail = sp_scheduler_queue_free_slot(queue);
if (tail == SIZE_MAX) {
*blocked_dims = record->klass == SP_SCHEDULER_CLASS_CONTROL
? SP_SCHEDULER_DIM_CONTROL_SLOTS
: SP_SCHEDULER_DIM_DATA_SLOTS;
return -1;
}
slot = &queue->records[tail];
*slot = *record;
slot->flow_index = flow_index;
slot->frame_len = frame_len;
slot->frame_offset = tail * queue->stride;
slot->in_use = 1;
memcpy(queue->arena + slot->frame_offset, frame, frame_len);
queue->order[queue->count] = (uint16_t)tail;
queue->count++;
queue->bytes += frame_len;
if (flow != NULL) {
flow->queued_records++;
}
if (flow != NULL && !flow->active) {
flow->active = 1;
flow->last_round = sched->round;
}
return 0;
}
/* Records the sequence a flow has committed, satisfying control dependencies. */
SP_TEST_API static void sp_scheduler_commit_seq(struct sp_scheduler_sched *sched, uint64_t flow,
uint64_t seq_abs) {
struct sp_scheduler_flow *entry = sp_scheduler_flow_lookup(sched, flow, 1);
if (entry != NULL && seq_abs > entry->committed_seq)
entry->committed_seq = seq_abs;
}
static int sp_scheduler_control_eligible(struct sp_scheduler_sched *sched,
const struct sp_scheduler_record *record) {
struct sp_scheduler_flow *flow;
if (record->depends_on_seq == 0)
return 1;
flow = sp_scheduler_flow_lookup(sched, record->flow, 0);
return flow != NULL && flow->committed_seq >= record->depends_on_seq;
}
/* Selects the next eligible control record, or SIZE_MAX when none is ready. */
static size_t sp_scheduler_next_control(struct sp_scheduler_sched *sched) {
size_t i;
for (i = 0; i < sched->control.count; ++i)
if (sp_scheduler_control_eligible(
sched, &sched->control.records[sched->control.order[i]]))
return i;
return SIZE_MAX;
}
/* Removes the record at FIFO position `position`, preserving the order of the
* rest. Only the small index array moves; frame bytes never do. */
static void sp_scheduler_queue_remove_at(struct sp_scheduler_sched *sched,
struct sp_scheduler_queue *queue,
size_t position) {
size_t slot = queue->order[position];
size_t i;
struct sp_scheduler_flow *flow = NULL;
if (queue->records[slot].klass == SP_SCHEDULER_CLASS_DATA ||
queue->records[slot].flow_index != UINT16_MAX)
flow = sp_scheduler_flow_lookup(sched, queue->records[slot].flow, 0);
queue->bytes -= queue->records[slot].frame_len;
if (flow != NULL && flow->queued_records > 0)
flow->queued_records--;
queue->records[slot].in_use = 0;
for (i = position + 1; i < queue->count; ++i)
queue->order[i - 1] = queue->order[i];
queue->count--;
}
/* A flow id is process-unique, but the scheduler's metadata arena is bounded
* to the maximum *concurrent* population. Retire both its queued records and
* DRR state when the live owner closes; otherwise sequential connections fill
* the arena permanently after SP_SCHEDULER_MAX_FLOWS admissions. */
static void sp_scheduler_retire_flow(struct sp_scheduler_sched *sched, uint64_t flow,
uint64_t generation) {
struct sp_scheduler_queue *queues[2] = {&sched->control, &sched->data};
int released_credit = 0;
size_t q;
size_t i;
if (flow == 0 || generation == 0)
return;
for (q = 0; q < 2; ++q) {
struct sp_scheduler_queue *queue = queues[q];
for (i = 0; i < queue->count;) {
const struct sp_scheduler_record *record =
&queue->records[queue->order[i]];
if (record->flow == flow && record->generation == generation) {
sp_scheduler_queue_remove_at(sched, queue, i);
released_credit = 1;
} else {
i++;
}
}
}
for (i = 0; i < sched->flow_count; ++i) {
if (sched->flows[i].flow != flow)
continue;
if (i + 1 < sched->flow_count)
memmove(&sched->flows[i], &sched->flows[i + 1],
(sched->flow_count - i - 1) * sizeof(sched->flows[0]));
memset(&sched->flows[--sched->flow_count], 0, sizeof(sched->flows[0]));
break;
}
if (released_credit)
sp_scheduler_publish_epoch(sched);
}
/* Grants one quantum to every active flow and advances the round. Carry is
* capped so an idle flow cannot bank unbounded credit, and a flow inactive for
* SP_SCHEDULER_DRR_IDLE_ROUNDS has its deficit reset outright. */
static void sp_scheduler_advance_round(struct sp_scheduler_sched *sched) {
size_t i;
sched->round++;
for (i = 0; i < sched->flow_count; ++i) {
struct sp_scheduler_flow *flow = &sched->flows[i];
if (!flow->active) {
if (sched->round - flow->last_round >= SP_SCHEDULER_DRR_IDLE_ROUNDS)
flow->deficit = 0;
continue;
}
flow->deficit += SP_SCHEDULER_DRR_QUANTUM;
if (flow->deficit > (int64_t)SP_SCHEDULER_DRR_CARRY_MAX)
flow->deficit = (int64_t)SP_SCHEDULER_DRR_CARRY_MAX;
flow->last_round = sched->round;
}
}
static int sp_scheduler_turn_exhausted(const struct sp_scheduler_turn *turn, uint64_t now_ns) {
if (turn->packets >= (size_t)SP_TUN_EGRESS_PKTS_TURN ||
turn->bytes >= (size_t)SP_TUN_EGRESS_BYTES_TURN ||
turn->records >= (size_t)SP_EFFECTS_TURN)
return 1;
return turn->started_ns != 0 && now_ns >= turn->started_ns &&
now_ns - turn->started_ns >= (uint64_t)SP_CALLBACK_BUDGET_NS;
}
static int sp_scheduler_write_record(struct sp_scheduler_queue *queue, size_t position,
sp_scheduler_write_fn write_fn, void *context,
struct sp_scheduler_turn *turn) {
const struct sp_scheduler_record *record = &queue->records[queue->order[position]];
unsigned packets = record->logical_packets == 0 ? 1 : record->logical_packets;
int result = write_fn(context, record, queue->arena + record->frame_offset);
/* A sink must return a verdict, rather than a frame byte count. */
assert(result >= SP_SCHEDULER_WRITE_FAILED);
assert(result <= SP_SCHEDULER_WRITE_RETIRED);
if (result <= SP_SCHEDULER_WRITE_BLOCKED) {
/* Never commit a blocked or failed write. The caller retries backpressure
* but propagates fatal failure without offering another record. */
turn->write_blocked = 1;
return result;
}
if (result == SP_SCHEDULER_WRITE_RETIRED)
return 1;
turn->packets += packets;
turn->bytes += record->frame_len;
turn->records++;
return 0;
}
/* Services one scheduler turn against a writable sink.
*
* Control is selected before bulk on every writability service, so a control
* record cannot be stuck behind data already queued. Data is then served by
* per-flow byte DRR: a whole record must fit the flow's current deficit, except
* for one explicitly charged atomic-frame exception per round for a record that
* can never fit a quantum. An issued batch never spans a future round.
*
* Returns the number of records written (excluding retirement), or SIZE_MAX
* on a fatal sink failure. */
SP_TEST_API static size_t sp_scheduler_service(struct sp_scheduler_sched *sched,
sp_scheduler_write_fn write_fn, void *context,
sp_scheduler_clock_fn clock_fn,
void *clock_context) {
struct sp_scheduler_turn turn;
uint64_t now_ns = clock_fn != NULL ? clock_fn(clock_context) : 0;
uint64_t round_started = now_ns;
size_t rounds = 0;
size_t written = 0;
memset(&turn, 0, sizeof(turn));
turn.started_ns = now_ns;
for (;;) {
size_t control_index;
size_t i;
int progressed = 0;
/* Re-read the clock each iteration; a by-value start time can never expire
* its own budget. */
if (clock_fn != NULL)
now_ns = clock_fn(clock_context);
if (sp_scheduler_turn_exhausted(&turn, now_ns))
break;
/* 1. Control and terminal responses first. */
control_index = sp_scheduler_next_control(sched);
if (control_index != SIZE_MAX) {
int write_rc = sp_scheduler_write_record(&sched->control, control_index,
write_fn, context, &turn);
if (write_rc == SP_SCHEDULER_WRITE_FAILED)
return SIZE_MAX;
if (write_rc < 0)
break;
sp_scheduler_queue_remove_at(sched, &sched->control, control_index);
sp_scheduler_publish_epoch(sched);
written += write_rc == 0;
continue;
}
/* 2. Bulk data under per-flow deficit. */
if (sched->data.count == 0)
break;
for (i = 0; i < sched->data.count; ++i) {
struct sp_scheduler_record *record = &sched->data.records[sched->data.order[i]];
struct sp_scheduler_flow *flow = sp_scheduler_flow_lookup(sched, record->flow, 0);
uint64_t charge;
if (flow == NULL)
continue;
charge = sp_scheduler_charge(record);
if ((uint64_t)flow->deficit < charge) {
/* One atomic-frame exception: a record that can never fit a full
* quantum is issued once and explicitly charged, so it cannot deadlock
* the flow. It is not a licence to borrow from a future round. */
if (charge > (uint64_t)SP_SCHEDULER_DRR_CARRY_MAX ||
charge <= (uint64_t)SP_SCHEDULER_DRR_QUANTUM ||
flow->deficit < (int64_t)SP_SCHEDULER_DRR_QUANTUM)
continue;
}
int write_rc = sp_scheduler_write_record(&sched->data, i, write_fn,
context, &turn);
if (write_rc == SP_SCHEDULER_WRITE_FAILED)
return SIZE_MAX;
if (write_rc < 0)
break;
if (write_rc == 0)
flow->deficit -= (int64_t)charge;
sp_scheduler_queue_remove_at(sched, &sched->data, i);
sp_scheduler_publish_epoch(sched);
written += write_rc == 0;
progressed = 1;
break;
}
if (turn.write_blocked)
break;
if (progressed)
continue;
/* No flow could afford its head record: advance the round. One logical
* round stops at SP_SCHED_ROUND_NS but resumes with ready ages unchanged. */
if (clock_fn != NULL && now_ns >= round_started &&
now_ns - round_started >= (uint64_t)SP_SCHED_ROUND_NS)
break;
if (sched->data.count == 0)
break;
/* Bounded independently of the clock: a caller holding time still must
* still see this call return. */
if (rounds >= (size_t)SP_SCHEDULER_ROUNDS_PER_CALL)
break;
rounds++;
sp_scheduler_advance_round(sched);
}
/* `queued_records` is maintained at admission/removal, so this state update
* is O(number of live scheduler flows), not a queue reconstruction scan. */
{
size_t i;
for (i = 0; i < sched->flow_count; ++i) {
struct sp_scheduler_flow *flow = &sched->flows[i];
flow->active = flow->queued_records != 0 ? 1 : 0;
}
}
return written;
}
/* Frozen overload response behavior.
*
* This is a classify-and-shed decision layer in front of the scheduler's
* control admission. It rate-limits response classes against the resource token
* buckets, coalesces equivalent current ACKs, refuses to answer a RST with a
* RST, sheds challenge/ICMP/new-admission work before ordinary responses when
* capacity runs out, and treats a terminal record that cannot be enqueued as a
* global-fatal invariant violation rather than backpressure. It is coupled to
* the scheduler on purpose: "retain the newest current ACK, never drop a
* terminal" is expressed through the scheduler's response and terminal slot
* reserves, and coalescing replaces an already-queued current ACK in place.
*
* The live adapter wires this in front of the live control lane; it is also exercised
* only on synthetic records. */
/* One token is NS_PER_S of allowance, so refill is exact integer arithmetic
* against an injected monotonic clock with no floating point. */
#define SP_SCHEDULER_TOKEN_NS 1000000000ULL
struct sp_scheduler_token_bucket {
uint64_t allowance;
uint64_t max_allowance;
uint64_t rate_per_s;
uint64_t last_ns;
};
static void sp_scheduler_bucket_init(struct sp_scheduler_token_bucket *bucket,
uint64_t rate_per_s, uint64_t burst,
uint64_t now_ns) {
bucket->rate_per_s = rate_per_s;
bucket->max_allowance = burst * SP_SCHEDULER_TOKEN_NS;
bucket->allowance = bucket->max_allowance;
bucket->last_ns = now_ns;
}
static void sp_scheduler_bucket_refill(struct sp_scheduler_token_bucket *bucket,
uint64_t now_ns) {
uint64_t elapsed;
uint64_t add;
uint64_t max_useful;
if (now_ns <= bucket->last_ns) {
/* A non-advancing or backward clock adds nothing and never leaks tokens. */
bucket->last_ns = now_ns;
return;
}
elapsed = now_ns - bucket->last_ns;
bucket->last_ns = now_ns;
if (bucket->rate_per_s == 0)
return;
/* Cap elapsed so the multiply cannot overflow; anything beyond this refills
* to the ceiling anyway. */
max_useful = bucket->max_allowance / bucket->rate_per_s + 1;
if (elapsed > max_useful)
elapsed = max_useful;
add = elapsed * bucket->rate_per_s;
if (bucket->max_allowance - bucket->allowance <= add)
bucket->allowance = bucket->max_allowance;
else
bucket->allowance += add;
}
/* Consumes one token if available. Returns 1 on success, 0 when the bucket is
* empty. */
static int sp_scheduler_bucket_take(struct sp_scheduler_token_bucket *bucket,
uint64_t now_ns) {
sp_scheduler_bucket_refill(bucket, now_ns);
if (bucket->allowance < SP_SCHEDULER_TOKEN_NS)
return 0;
bucket->allowance -= SP_SCHEDULER_TOKEN_NS;
return 1;
}
enum sp_scheduler_response_kind {
SP_SCHEDULER_RESP_CURRENT_ACK = 0,
SP_SCHEDULER_RESP_CHALLENGE_ACK,
SP_SCHEDULER_RESP_CLOSED_RST,
SP_SCHEDULER_RESP_ICMP,
SP_SCHEDULER_RESP_TERMINAL,
};
enum sp_scheduler_shed_decision {
SP_SCHEDULER_SHED_ADMIT = 0, /* enqueued for transmission */
SP_SCHEDULER_SHED_COALESCED, /* replaced an equivalent current ACK in place */
SP_SCHEDULER_SHED_DROPPED, /* rate/capacity shed, counted, non-fatal */
SP_SCHEDULER_SHED_RST_TO_RST, /* CLOSED reset suppressed: never answer RST with RST */
SP_SCHEDULER_SHED_FATAL, /* a terminal record could not be enqueued */
};
enum { SP_SCHEDULER_OVERLOAD_MAX_FLOWS = SP_SCHEDULER_MAX_FLOWS };
struct sp_scheduler_overload_flow {
uint64_t flow;
struct sp_scheduler_token_bucket challenge;
struct sp_scheduler_token_bucket icmp;
uint8_t in_use;
};
struct sp_scheduler_overload {
struct sp_scheduler_token_bucket challenge_all;
struct sp_scheduler_token_bucket closed_rst_all;
struct sp_scheduler_token_bucket icmp_all;
struct sp_scheduler_overload_flow flows[SP_SCHEDULER_OVERLOAD_MAX_FLOWS];
/* Saturating counters with fixed reason labels. */
uint64_t coalesced_current_ack;
uint64_t shed_challenge;
uint64_t shed_closed_rst;
uint64_t shed_icmp;
uint64_t shed_current_ack;
uint64_t suppressed_rst_to_rst;
uint64_t fatal_terminal;
};
static void sp_scheduler_sat_inc(uint64_t *counter) {
if (*counter != UINT64_MAX)
(*counter)++;
}
/* `ov` must be zero-filled, as for sp_scheduler_init(). */
SP_TEST_API static void sp_scheduler_overload_init(struct sp_scheduler_overload *ov,
uint64_t now_ns) {
sp_scheduler_bucket_init(&ov->challenge_all, SP_CHALLENGE_ALL_PER_S,
SP_CHALLENGE_ALL_BURST, now_ns);
sp_scheduler_bucket_init(&ov->closed_rst_all, SP_CLOSED_RST_ALL_PER_S,
SP_CLOSED_RST_ALL_BURST, now_ns);
sp_scheduler_bucket_init(&ov->icmp_all, SP_ICMP_ALL_PER_S, SP_ICMP_ALL_BURST,
now_ns);
}
static struct sp_scheduler_overload_flow *
sp_scheduler_overload_flow(struct sp_scheduler_overload *ov, uint64_t flow, uint64_t now_ns) {
size_t i;
size_t free_slot = SP_SCHEDULER_OVERLOAD_MAX_FLOWS;
for (i = 0; i < SP_SCHEDULER_OVERLOAD_MAX_FLOWS; ++i) {
if (ov->flows[i].in_use && ov->flows[i].flow == flow)
return &ov->flows[i];
if (!ov->flows[i].in_use && free_slot == SP_SCHEDULER_OVERLOAD_MAX_FLOWS)
free_slot = i;
}
if (free_slot == SP_SCHEDULER_OVERLOAD_MAX_FLOWS)
return NULL;
ov->flows[free_slot].flow = flow;
ov->flows[free_slot].in_use = 1;
sp_scheduler_bucket_init(&ov->flows[free_slot].challenge, SP_CHALLENGE_FLOW_PER_S,
SP_CHALLENGE_FLOW_BURST, now_ns);
sp_scheduler_bucket_init(&ov->flows[free_slot].icmp, SP_ICMP_FLOW_PER_S,
SP_ICMP_FLOW_BURST, now_ns);
return &ov->flows[free_slot];
}
static void sp_scheduler_overload_retire_flow(struct sp_scheduler_overload *ov,
uint64_t flow) {
size_t i;
if (flow == 0)
return;
for (i = 0; i < SP_SCHEDULER_OVERLOAD_MAX_FLOWS; ++i) {
if (ov->flows[i].in_use && ov->flows[i].flow == flow) {
memset(&ov->flows[i], 0, sizeof(ov->flows[i]));
return;
}
}
}
/* Finds a queued current ACK for a flow so an equivalent one can coalesce onto
* it, retaining the newest. Returns the control FIFO position or SIZE_MAX. */
static size_t sp_scheduler_find_current_ack(struct sp_scheduler_sched *sched, uint64_t flow) {
size_t i;
for (i = 0; i < sched->control.count; ++i) {
struct sp_scheduler_record *record =
&sched->control.records[sched->control.order[i]];
if (record->flow == flow && record->tier == SP_SCHEDULER_CTRL_RESPONSE &&
!record->terminal)
return i;
}
return SIZE_MAX;
}
static enum sp_scheduler_ctrl_tier sp_scheduler_response_tier(enum sp_scheduler_response_kind kind) {
switch (kind) {
case SP_SCHEDULER_RESP_TERMINAL:
return SP_SCHEDULER_CTRL_TERMINAL;
case SP_SCHEDULER_RESP_CURRENT_ACK:
return SP_SCHEDULER_CTRL_RESPONSE;
case SP_SCHEDULER_RESP_CHALLENGE_ACK:
case SP_SCHEDULER_RESP_CLOSED_RST:
case SP_SCHEDULER_RESP_ICMP:
default:
return SP_SCHEDULER_CTRL_ORDINARY;
}
}
/* Classifies one outbound control response and either admits it to the
* scheduler, coalesces it, sheds it, or reports a terminal-enqueue fatal.
*
* `in_reply_to_rst` matters only for CLOSED resets: a reset is never emitted in
* answer to an incoming RST. On ADMIT the record is placed on the scheduler's
* control lane at the class-appropriate reserve tier. */
static enum sp_scheduler_shed_decision
sp_scheduler_overload_response_with_dependency_index(
struct sp_scheduler_overload *ov, struct sp_scheduler_sched *sched,
enum sp_scheduler_response_kind kind, uint64_t flow, uint64_t generation,
int in_reply_to_rst, const uint8_t *frame, size_t frame_len,
uint64_t seq_abs, uint64_t depends_on_seq, uint64_t now_ns,
uint16_t flow_index) {
struct sp_scheduler_overload_flow *ovflow;
struct sp_scheduler_record record;
unsigned blocked = 0;
enum sp_scheduler_ctrl_tier tier = sp_scheduler_response_tier(kind);
/* Terminals are never rate-limited or shed. Failure to enqueue one means
* ownership can never be released, which is a fatal invariant violation. */
if (kind == SP_SCHEDULER_RESP_TERMINAL) {
memset(&record, 0, sizeof(record));
record.flow = flow;
record.generation = generation;
record.flow_index = flow_index;
record.klass = SP_SCHEDULER_CLASS_CONTROL;
record.tier = SP_SCHEDULER_CTRL_TERMINAL;
record.terminal = 1;
record.seq_abs = seq_abs;
record.depends_on_seq = depends_on_seq;
record.logical_bytes = frame_len;
record.logical_packets = 1;
if (sp_scheduler_admit_with_flow_index(sched, &record, frame, frame_len,
&blocked, flow_index) < 0) {
sp_scheduler_sat_inc(&ov->fatal_terminal);
return SP_SCHEDULER_SHED_FATAL;
}
return SP_SCHEDULER_SHED_ADMIT;
}
/* A CLOSED reset is never a reply to a RST. */
if (kind == SP_SCHEDULER_RESP_CLOSED_RST && in_reply_to_rst) {
sp_scheduler_sat_inc(&ov->suppressed_rst_to_rst);
return SP_SCHEDULER_SHED_RST_TO_RST;
}
/* Equivalent current ACKs coalesce to one per flow per input batch and are
* not charged to the challenge bucket. */
if (kind == SP_SCHEDULER_RESP_CURRENT_ACK) {
size_t existing = sp_scheduler_find_current_ack(sched, flow);
if (existing != SIZE_MAX) {
struct sp_scheduler_record *slot =
&sched->control.records[sched->control.order[existing]];
/* Retain the newest: overwrite the queued frame in place. */
if (frame_len <= sched->control.stride) {
memcpy(sched->control.arena + slot->frame_offset, frame, frame_len);
sched->control.bytes -= slot->frame_len;
sched->control.bytes += frame_len;
slot->frame_len = frame_len;
slot->seq_abs = seq_abs;
}
sp_scheduler_sat_inc(&ov->coalesced_current_ack);
return SP_SCHEDULER_SHED_COALESCED;
}
}
/* Only challenge ACK and ICMP policy has a per-flow bucket. Current ACK and
* CLOSED RST traffic must not retain historical auxiliary tuple ids. */
ovflow = NULL;
if (kind == SP_SCHEDULER_RESP_CHALLENGE_ACK) {
ovflow = sp_scheduler_overload_flow(ov, flow, now_ns);
if (ovflow == NULL || !sp_scheduler_bucket_take(&ovflow->challenge, now_ns) ||
!sp_scheduler_bucket_take(&ov->challenge_all, now_ns)) {
sp_scheduler_sat_inc(&ov->shed_challenge);
return SP_SCHEDULER_SHED_DROPPED;
}
} else if (kind == SP_SCHEDULER_RESP_CLOSED_RST) {
if (!sp_scheduler_bucket_take(&ov->closed_rst_all, now_ns)) {
sp_scheduler_sat_inc(&ov->shed_closed_rst);
return SP_SCHEDULER_SHED_DROPPED;
}
} else if (kind == SP_SCHEDULER_RESP_ICMP) {
ovflow = sp_scheduler_overload_flow(ov, flow, now_ns);
if (ovflow == NULL || !sp_scheduler_bucket_take(&ovflow->icmp, now_ns) ||
!sp_scheduler_bucket_take(&ov->icmp_all, now_ns)) {
sp_scheduler_sat_inc(&ov->shed_icmp);
return SP_SCHEDULER_SHED_DROPPED;
}
}
memset(&record, 0, sizeof(record));
record.flow = flow;
record.generation = generation;
record.flow_index = flow_index;
record.klass = SP_SCHEDULER_CLASS_CONTROL;
record.tier = tier;
record.seq_abs = seq_abs;
record.logical_bytes = frame_len;
record.logical_packets = 1;
if (sp_scheduler_admit_with_flow_index(sched, &record, frame, frame_len, &blocked,
flow_index) < 0) {
/* Capacity exhausted: current ACKs are retained via their reserve where
* possible; everything else is shed. A shed here is never fatal. */
if (kind == SP_SCHEDULER_RESP_CURRENT_ACK)
sp_scheduler_sat_inc(&ov->shed_current_ack);
else if (kind == SP_SCHEDULER_RESP_CHALLENGE_ACK)
sp_scheduler_sat_inc(&ov->shed_challenge);
else if (kind == SP_SCHEDULER_RESP_CLOSED_RST)
sp_scheduler_sat_inc(&ov->shed_closed_rst);
else
sp_scheduler_sat_inc(&ov->shed_icmp);
return SP_SCHEDULER_SHED_DROPPED;
}
return SP_SCHEDULER_SHED_ADMIT;
}
static enum sp_scheduler_shed_decision sp_scheduler_overload_response_with_dependency(
struct sp_scheduler_overload *ov, struct sp_scheduler_sched *sched,
enum sp_scheduler_response_kind kind, uint64_t flow, uint64_t generation,
int in_reply_to_rst, const uint8_t *frame, size_t frame_len,
uint64_t seq_abs, uint64_t depends_on_seq, uint64_t now_ns) {
return sp_scheduler_overload_response_with_dependency_index(
ov, sched, kind, flow, generation, in_reply_to_rst, frame, frame_len,
seq_abs, depends_on_seq, now_ns, UINT16_MAX);
}
static uint16_t sp_icmp_reply_id;
static uint64_t sp_ns_now_ns(void);
/* Build the bounded ICMP destination-unreachable product used by UDP refusal
* and PMTU reporting. The quote is copied from an executor-validated value;
* no transition code parses mutable packet storage. */
SP_TEST_API static enum sp_codec_status
sp_build_icmp_control_reply(const struct sp_icmp_control_request *request,
uint16_t ipv4_id,
struct sp_scalar_packet *reply) {
struct sp_scalar_packet candidate;
if (request == NULL || reply == NULL)
return SP_CODEC_SHORT;
if (request->code != 3 && request->code != 4)
return SP_CODEC_PROTOCOL;
if ((request->code == 3 && request->mtu != 0) ||
(request->code == 4 && request->mtu == 0))
return SP_CODEC_LENGTH;
if (request->quote[0] != 0x45 || request->quote[9] != IPPROTO_UDP ||
sp_get_be16(request->quote + 2) <
SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES ||
(sp_get_be16(request->quote + 6) & (IP_MF | IP_OFFMASK)) != 0)
return SP_CODEC_PROTOCOL;
memset(&candidate, 0, sizeof(candidate));
candidate.bytes[0] = 0x45;
sp_put_be16(candidate.bytes + 2, 56);
sp_put_be16(candidate.bytes + 4, ipv4_id);
candidate.bytes[8] = 64;
candidate.bytes[9] = IPPROTO_ICMP;
memcpy(candidate.bytes + 12, request->quote + 16, 4);
memcpy(candidate.bytes + 16, request->quote + 12, 4);
candidate.bytes[20] = 3;
candidate.bytes[21] = request->code;
if (request->code == 4)
sp_put_be16(candidate.bytes + 26, request->mtu);
memcpy(candidate.bytes + 28, request->quote,
SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES);
sp_put_be16(candidate.bytes + 22,
sp_checksum_complete(candidate.bytes + 20, 36));
sp_rechecksum_ipv4(candidate.bytes);
candidate.len = 56;
*reply = candidate;
return SP_CODEC_OK;
}
/* Role classification consumes only the sealed policy value. Exact live-flow
* and DNS-mapping checks remain in their role handlers, but no CLI table or
* mutable runtime object is consulted here. */
static int sp_policy_child_destination_allowed(
const struct sp_net_policy_snapshot *policy,
const struct sp_packet_value *parsed) {
uint16_t port;
uint32_t destination_be = htonl(parsed->ip.dst);
if (policy == NULL || !policy->initialized)
return 0;
if (parsed->ip.protocol == IPPROTO_TCP)
port = parsed->tcp.dst_port;
else if (parsed->ip.protocol == IPPROTO_UDP)
port = parsed->udp.dst_port;
else
return 0;
if (destination_be == PUBLISH_SYNTHETIC_IP)
return 1;
if (is_gateway_ip(destination_be))
return sp_policy_gateway_allowed(policy, destination_be, port,
parsed->ip.protocol);
if (parsed->ip.protocol == IPPROTO_UDP && destination_be == DNS_PROXY_IP)
return policy->egress == EGRESS_SOCKS && policy->socks_remote_dns &&
port == 53;
if (policy->egress == EGRESS_NONE)
return 0;
if (policy->egress == EGRESS_DIRECT)
return sp_policy_direct_egress_allowed(policy, destination_be, port,
parsed->ip.protocol);
return parsed->ip.dst != 0 && parsed->ip.dst != UINT32_MAX &&
(parsed->ip.dst >> 24) != 0 && (parsed->ip.dst >> 24) != 127 &&
(parsed->ip.dst >> 28) != 0xe && (parsed->ip.dst >> 28) != 0xf;
}
static inline uint32_t tcp_seq_add_len(uint32_t seq, size_t len) {
return seq + (uint32_t)len;
}
static int ipv4_is_loopback(uint32_t ip_be) {
return (ntohl(ip_be) & 0xff000000U) == 0x7f000000U;
}
static int sp_publish_synthetic_port_in_use(uint16_t port);
/* Resolves a publish entry of the UDP table to its live flow, or NULL. */
static struct publish_udp_flow *publish_udp_flow_for_key(const struct sp_udp_key *key) {
struct sp_udp_entry *entry = sp_udp_table_lookup_entry(key);
struct publish_udp_flow *f;
if (entry == NULL || entry->value.role != SP_UDP_ROLE_PUBLISH ||
entry->value.index >= MAX_PUBLISH_UDP)
return NULL;
f = &publish_udp_flows[entry->value.index];
return f->used && f->generation == entry->generation ? f : NULL;
}
static struct publish_udp_flow *publish_udp_find(uint16_t synthetic_port,
uint16_t child_port) {
struct sp_udp_key key;
struct publish_udp_flow *f;
sp_udp_publish_child_key(&key, synthetic_port, child_port);
f = publish_udp_flow_for_key(&key);
if (f == NULL || f->core.role != SP_UDP_ROLE_PUBLISH ||
f->synthetic_port != synthetic_port || f->child_port != child_port)
return NULL;
return f;
}
static struct publish_udp_flow *publish_udp_find_host_peer(
int rule_index, uint32_t local_ip, uint16_t local_port,
const struct sockaddr_in *host_peer) {
struct sp_udp_key key;
struct publish_udp_flow *f;
sp_udp_publish_key(&key, rule_index, local_ip, local_port, host_peer);
f = publish_udp_flow_for_key(&key);
if (f == NULL || f->rule_index != rule_index || f->published_local_ip != local_ip ||
f->host_peer.sin_addr.s_addr != host_peer->sin_addr.s_addr ||
f->host_peer.sin_port != host_peer->sin_port)
return NULL;
return f;
}
static struct publish_udp_flow *publish_udp_alloc(void) {
for (int i = 0; i < MAX_PUBLISH_UDP; ++i) {
if (!publish_udp_flows[i].used) {
uint32_t generation =
epoll_next_generation(publish_udp_flows[i].generation);
memset(&publish_udp_flows[i], 0, sizeof(publish_udp_flows[i]));
publish_udp_flows[i].generation = generation;
publish_udp_flows[i].used = 1;
(void)sp_udp_runtime_step(&publish_udp_flows[i].core, SP_UDP_BEGIN, 0,
SP_UDP_ROLE_PUBLISH);
return &publish_udp_flows[i];
}
}
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return NULL;
}
static void publish_udp_update_rule_events(int rule_index) {
struct publish_rule *rule;
uint32_t events = EPOLLIN | EPOLLERR | EPOLLHUP;
if (rule_index < 0 || rule_index >= publish_rule_count || g_epfd < 0)
return;
rule = &publish_rules[rule_index];
if (rule->proto != IPPROTO_UDP || rule->listen_fd < 0)
return;
for (size_t i = 0; i < MAX_PUBLISH_UDP; ++i) {
if (publish_udp_flows[i].used &&
publish_udp_flows[i].rule_index == rule_index &&
publish_udp_flows[i].core.pending_count != 0) {
events |= EPOLLOUT;
break;
}
}
{
struct epoll_event ev = {
.events = events,
.data.u64 = epoll_registration_token(
FD_PUBLISH_UDP_SOCKET, (size_t)rule_index, rule->listen_fd, 1)};
if (sp_epoll_update(EPOLL_CTL_MOD, rule->listen_fd, &ev,
&rule->interest) < 0) {
sp_epoll_update_failed("publish UDP listener");
g_broker_io_failed = 1;
}
}
}
static void publish_udp_close_flow(struct publish_udp_flow *f) {
struct sp_udp_key external_key;
struct sp_udp_key child_key;
uint32_t generation;
uint16_t local_port = 0;
if (f == NULL)
return;
generation = f->generation;
if (f->rule_index >= 0 && f->rule_index < publish_rule_count)
local_port = publish_rules[f->rule_index].host_port;
if (f->used) {
sp_udp_publish_key(&external_key, f->rule_index, f->published_local_ip,
local_port, &f->host_peer);
sp_udp_publish_child_key(&child_key, f->synthetic_port, f->child_port);
sp_udp_table_remove(&external_key, f->generation);
sp_udp_table_remove(&child_key, f->generation);
}
if (f->admission_held)
sp_resource_admission_release_source(
f->host_peer.sin_addr.s_addr,
sp_udp_admission_role(SP_UDP_ROLE_PUBLISH));
if (f->rule_admission_held)
sp_resource_admission_release_source((uint32_t)f->rule_index,
SP_UDP_ADMISSION_RULE);
if (f->global_admission_held)
sp_resource_admission_release_source(0, SP_UDP_ADMISSION_GLOBAL);
(void)sp_udp_runtime_step(&f->core, SP_UDP_CLOSE, 0, 0);
while (f->core.pending_count != 0) {
uint16_t slot = f->core.pending_slots[f->core.pending_head];
udp_pending_release(slot);
sp_udp_pop(&f->core);
}
{
int rule_index = f->rule_index;
memset(f, 0, sizeof(*f));
f->generation = generation;
f->rule_index = -1;
for (size_t i = 0; i < SP_UDP_PENDING_PER_FLOW; ++i)
f->core.pending_slots[i] = UINT16_MAX;
publish_udp_update_rule_events(rule_index);
}
}
/* Synthetic source port for a published flow (TCP and UDP share the pool). */
static uint16_t sp_ns_publish_port_alloc(void) {
unsigned int span = PUBLISH_SYNTH_PORT_MAX - PUBLISH_SYNTH_PORT_MIN + 1U;
for (unsigned int attempt = 0; attempt < SP_PORT_ALLOC_PROBES; ++attempt) {
uint64_t mixed =
(uint64_t)next_publish_synth_port++ +
(uint64_t)attempt * UINT64_C(0x9e3779b97f4a7c15) +
g_ns_aux_hash_key;
uint16_t port;
if (next_publish_synth_port > PUBLISH_SYNTH_PORT_MAX)
next_publish_synth_port = PUBLISH_SYNTH_PORT_MIN;
mixed ^= mixed >> 30;
mixed *= UINT64_C(0xbf58476d1ce4e5b9);
mixed ^= mixed >> 27;
mixed *= UINT64_C(0x94d049bb133111eb);
mixed ^= mixed >> 31;
port = (uint16_t)(PUBLISH_SYNTH_PORT_MIN + mixed % span);
if (!sp_publish_synthetic_port_in_use(port))
return port;
}
return 0;
}
/* Send a TCP RST packet */
static void send_tcp_rst(int tunfd, uint32_t saddr, uint32_t daddr,
uint16_t sport, uint16_t dport, uint32_t seq,
uint32_t ack_seq) {
uint8_t record[SP_VNET_HDR_BYTES + 40];
size_t record_len = sp_vnet_build_tcp_record(
record, sizeof(record), ntohl(saddr), ntohl(daddr), sport, dport, seq,
ack_seq, SP_TCP_FLAG_RST | SP_TCP_FLAG_ACK, NULL, 0, 0);
if (record_len == 0) {
DBG("failed to construct TCP RST");
return;
}
IGNORE_RESULT(tun_write_frame(tunfd, record, record_len, "TCP RST"));
}
struct sp_publish_turn {
size_t items;
size_t bytes;
uint64_t started_ns;
};
/* Published sockets are level-triggered. Returning at any one of these
* limits leaves readiness asserted, so epoll resumes the source in a later
* turn while TUN, timers, and other published rules get service. */
static int sp_publish_turn_exhausted(const struct sp_publish_turn *turn,
size_t item_limit, size_t byte_limit,
uint64_t now_ns) {
if (turn->items >= item_limit ||
(byte_limit != 0 && turn->bytes >= byte_limit))
return 1;
return turn->started_ns != 0 && now_ns >= turn->started_ns &&
now_ns - turn->started_ns >= (uint64_t)SP_CALLBACK_BUDGET_NS;
}
static void sp_publish_turn_account(struct sp_publish_turn *turn,
size_t bytes) {
if (turn->items != SIZE_MAX)
turn->items++;
if (SIZE_MAX - turn->bytes < bytes)
turn->bytes = SIZE_MAX;
else
turn->bytes += bytes;
}
static void handle_publish_tcp_listener(int tunfd, struct publish_rule *rule) {
struct sp_publish_turn turn = {.started_ns = sp_ns_now_ns()};
while (!sp_publish_turn_exhausted(
&turn, SP_PUBLISH_ACCEPTS_TURN, 0, sp_ns_now_ns())) {
struct sockaddr_in peer;
socklen_t peer_len = sizeof(peer);
int fd = accept4(rule->listen_fd, (struct sockaddr *)&peer, &peer_len,
SOCK_CLOEXEC | SOCK_NONBLOCK);
if (fd < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK)
return;
return;
}
sp_publish_turn_account(&turn, 0);
const char *option_reason = "unknown";
/* Already established from accept4(), so unlike the connect paths the
* SO_RCVBUF resize cannot influence the negotiated window scale. */
if (sp_scheduler_configure_host_tcp_socket(fd, &option_reason) < 0) {
DBG("accepted publish socket rejected: %s (%s)", option_reason,
strerror(errno));
close(fd);
continue;
}
peer_len = sizeof(peer);
if (getpeername(fd, (struct sockaddr *)&peer, &peer_len) < 0 ||
peer.sin_family != AF_INET ||
!ipv4_is_loopback(peer.sin_addr.s_addr)) {
close(fd);
continue;
}
if (sp_ns_accept_publish(tunfd, rule, fd, &peer) < 0) {
close(fd);
return;
}
}
}
static int sp_publish_udp_mmsg_entry(unsigned int vector_index,
const struct publish_udp_flow *flow,
const struct sp_udp_pending_slot *pending,
uint16_t slot) {
struct msghdr *message = &g_udp_mmsg_tx_messages[vector_index].msg_hdr;
struct cmsghdr *cmsg;
struct in_pktinfo pktinfo;
if (vector_index >= SP_UDP_MMSG_MAX || slot >= SP_UDP_PENDING_ALL)
return -1;
g_udp_mmsg_tx_iov[vector_index].iov_base = udp_pending_data(slot);
g_udp_mmsg_tx_iov[vector_index].iov_len = pending->length;
g_udp_mmsg_tx_names[vector_index] = flow->host_peer;
message->msg_name = &g_udp_mmsg_tx_names[vector_index];
message->msg_namelen = sizeof(g_udp_mmsg_tx_names[vector_index]);
message->msg_control = g_udp_mmsg_tx_control[vector_index];
message->msg_controllen = CMSG_SPACE(sizeof(pktinfo));
memset(&pktinfo, 0, sizeof(pktinfo));
pktinfo.ipi_spec_dst.s_addr = flow->published_local_ip;
pktinfo.ipi_addr.s_addr = flow->published_local_ip;
cmsg = CMSG_FIRSTHDR(message);
if (cmsg == NULL)
return -1;
cmsg->cmsg_level = IPPROTO_IP;
cmsg->cmsg_type = IP_PKTINFO;
cmsg->cmsg_len = CMSG_LEN(sizeof(pktinfo));
memcpy(CMSG_DATA(cmsg), &pktinfo, sizeof(pktinfo));
return 0;
}
static void sp_publish_udp_mark_progress(struct publish_udp_flow *flow) {
uint64_t now_ns = sp_ns_now_ns();
(void)sp_udp_runtime_step(&flow->core, SP_UDP_PROGRESS, now_ns, 0);
}
static void flush_publish_udp_rule(struct publish_rule *rule) {
struct sp_publish_turn turn = {.started_ns = sp_ns_now_ns()};
while (!sp_publish_turn_exhausted(
&turn, SP_PUBLISH_DATAGRAMS_TURN, SP_PUBLISH_BYTES_TURN,
sp_ns_now_ns())) {
struct publish_udp_flow *owners[SP_UDP_MMSG_MAX];
uint16_t slots[SP_UDP_MMSG_MAX];
unsigned int count = 0;
size_t remaining = SP_PUBLISH_DATAGRAMS_TURN - turn.items;
unsigned int limit =
remaining > SP_UDP_MMSG_MAX ? SP_UDP_MMSG_MAX : (unsigned int)remaining;
int result;
int invariant_failed = 0;
sp_udp_mmsg_tx_reset(limit);
for (size_t i = 0; i < MAX_PUBLISH_UDP && count < limit; ++i) {
struct publish_udp_flow *flow = &publish_udp_flows[i];
/* Closing a flow releases its slots, so entries already gathered for
* it must leave the vector too. */
unsigned int flow_first = count;
if (!flow->used || flow->rule_index != rule->rule_index)
continue;
for (unsigned int q = 0; q < flow->core.pending_count && count < limit; ++q) {
size_t queue_index =
(flow->core.pending_head + q) % SP_UDP_PENDING_PER_FLOW;
uint16_t slot = flow->core.pending_slots[queue_index];
struct sp_udp_pending_slot *pending;
if (slot >= SP_UDP_PENDING_ALL) {
publish_udp_close_flow(flow);
count = flow_first;
break;
}
pending = &g_udp_pending_slots[slot];
if (!pending->used || !pending->owner_publish ||
pending->generation != flow->generation ||
pending->owner_index != (uint16_t)i ||
sp_publish_udp_mmsg_entry(count, flow, pending, slot) < 0) {
publish_udp_close_flow(flow);
count = flow_first;
break;
}
owners[count] = flow;
slots[count] = slot;
count++;
}
}
if (count == 0) {
publish_udp_update_rule_events(rule->rule_index);
return;
}
result =
sp_udp_sendmmsg_one_fd(rule->listen_fd, count, turn.started_ns);
if (result == 0) {
publish_udp_close_flow(owners[0]);
publish_udp_update_rule_events(rule->rule_index);
return;
}
if (result < 0) {
struct publish_udp_flow *flow = owners[0];
uint16_t slot = slots[0];
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
publish_udp_update_rule_events(rule->rule_index);
return;
}
if (errno == EMSGSIZE || errno == ECONNREFUSED || errno == ENOBUFS ||
errno == ENOMEM) {
if (flow->core.pending_count == 0 ||
flow->core.pending_slots[flow->core.pending_head] != slot) {
publish_udp_close_flow(flow);
return;
}
sp_counter_inc(&g_resource_counters.udp_terminal_drops);
udp_pending_release(slot);
sp_udp_pop(&flow->core);
continue;
}
publish_udp_close_flow(flow);
publish_udp_update_rule_events(rule->rule_index);
return;
}
for (int i = 0; i < result; ++i) {
struct publish_udp_flow *flow = owners[i];
uint16_t slot = slots[i];
size_t length = g_udp_pending_slots[slot].length;
if (!flow->used || flow->core.pending_count == 0 ||
flow->core.pending_slots[flow->core.pending_head] != slot) {
if (flow->used)
publish_udp_close_flow(flow);
invariant_failed = 1;
continue;
}
if (g_udp_mmsg_tx_messages[i].msg_len !=
g_udp_mmsg_tx_iov[i].iov_len)
invariant_failed = 1;
udp_pending_release(slot);
sp_udp_pop(&flow->core);
sp_publish_turn_account(&turn, length);
sp_publish_udp_mark_progress(flow);
}
if (invariant_failed || (unsigned int)result < count) {
publish_udp_update_rule_events(rule->rule_index);
return;
}
}
publish_udp_update_rule_events(rule->rule_index);
}
static int sp_publish_udp_pktinfo_valid(const struct msghdr *msg,
uint32_t expected_local_ip) {
unsigned int count = 0;
if (msg == NULL || (msg->msg_flags & (MSG_TRUNC | MSG_CTRUNC)) != 0)
return 0;
for (struct cmsghdr *cmsg = CMSG_FIRSTHDR((struct msghdr *)msg);
cmsg != NULL; cmsg = sp_cmsg_next((struct msghdr *)msg, cmsg)) {
struct in_pktinfo pktinfo;
if (cmsg->cmsg_level != IPPROTO_IP || cmsg->cmsg_type != IP_PKTINFO ||
cmsg->cmsg_len != CMSG_LEN(sizeof(pktinfo)))
return 0;
memcpy(&pktinfo, CMSG_DATA(cmsg), sizeof(pktinfo));
if (pktinfo.ipi_addr.s_addr != expected_local_ip)
return 0;
count++;
}
return count == 1;
}
static void handle_publish_udp_datagram(int tunfd, struct publish_rule *rule,
const struct sockaddr_in *peer,
const uint8_t *payload,
size_t payload_len) {
struct publish_udp_flow *f = publish_udp_find_host_peer(
rule->rule_index, rule->host_ip, rule->host_port, peer);
if (f == NULL) {
uint16_t synthetic_port = sp_ns_publish_port_alloc();
struct sp_udp_key external_key;
struct sp_udp_key child_key;
if (synthetic_port == 0)
return;
if (sp_udp_admit_source(peer->sin_addr.s_addr, SP_UDP_ROLE_PUBLISH) < 0) {
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
if (sp_udp_admit_rule(rule->rule_index) < 0) {
sp_resource_admission_release_source(
peer->sin_addr.s_addr, sp_udp_admission_role(SP_UDP_ROLE_PUBLISH));
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
if (sp_udp_admit_global() < 0) {
sp_resource_admission_release_source(
peer->sin_addr.s_addr, sp_udp_admission_role(SP_UDP_ROLE_PUBLISH));
sp_resource_admission_release_source((uint32_t)rule->rule_index,
SP_UDP_ADMISSION_RULE);
sp_counter_inc(&g_resource_counters.udp_flow_rejections);
return;
}
f = publish_udp_alloc();
if (f == NULL) {
sp_resource_admission_release_source(
peer->sin_addr.s_addr, sp_udp_admission_role(SP_UDP_ROLE_PUBLISH));
sp_resource_admission_release_source((uint32_t)rule->rule_index,
SP_UDP_ADMISSION_RULE);
sp_resource_admission_release_source(0, SP_UDP_ADMISSION_GLOBAL);
return;
}
f->admission_held = 1;
f->rule_admission_held = 1;
f->global_admission_held = 1;
f->rule_index = rule->rule_index;
f->host_peer = *peer;
f->published_local_ip = rule->host_ip;
f->synthetic_port = synthetic_port;
f->child_port = rule->container_port;
(void)sp_udp_runtime_step(&f->core, SP_UDP_READY, sp_ns_now_ns(), 0);
sp_udp_publish_key(&external_key, f->rule_index, f->published_local_ip,
rule->host_port, &f->host_peer);
sp_udp_publish_child_key(&child_key, f->synthetic_port, f->child_port);
if (sp_udp_table_insert(&external_key, f->generation,
(uint16_t)(f - publish_udp_flows), f->core.role) <
0 ||
sp_udp_table_insert(&child_key, f->generation,
(uint16_t)(f - publish_udp_flows), f->core.role) <
0) {
sp_udp_table_remove(&external_key, f->generation);
publish_udp_close_flow(f);
return;
}
}
{
uint64_t now_ns = sp_ns_now_ns();
(void)sp_udp_runtime_step(&f->core, SP_UDP_PROGRESS, now_ns, 0);
}
sp_udp_emit_tun(tunfd, ntohl(PUBLISH_SYNTHETIC_IP), ntohl(SANDBOX_IP),
f->synthetic_port, f->child_port, payload, payload_len,
"publish UDP inject");
}
static void handle_publish_udp_socket(int tunfd, struct publish_rule *rule) {
struct sp_publish_turn turn = {.started_ns = sp_ns_now_ns()};
while (!sp_publish_turn_exhausted(
&turn, SP_PUBLISH_DATAGRAMS_TURN, SP_PUBLISH_BYTES_TURN,
sp_ns_now_ns())) {
size_t remaining = SP_PUBLISH_DATAGRAMS_TURN - turn.items;
unsigned int requested =
remaining > SP_UDP_MMSG_MAX ? SP_UDP_MMSG_MAX : (unsigned int)remaining;
int received =
sp_udp_recvmmsg_one_fd(rule->listen_fd, requested, turn.started_ns);
if (received <= 0)
return;
for (int i = 0; i < received; ++i) {
struct msghdr *message = &g_udp_mmsg_messages[i].msg_hdr;
size_t payload_len = g_udp_mmsg_messages[i].msg_len;
sp_publish_turn_account(&turn, payload_len);
if (sp_udp_mmsg_slot_basic_valid((unsigned int)i) &&
payload_len <= SP_UDP_PAYLOAD_MAX &&
ipv4_is_loopback(g_udp_mmsg_sources[i].sin_addr.s_addr) &&
sp_publish_udp_pktinfo_valid(message, rule->host_ip))
handle_publish_udp_datagram(tunfd, rule, &g_udp_mmsg_sources[i],
g_udp_mmsg_payload[i], payload_len);
if (sp_publish_turn_exhausted(
&turn, SP_PUBLISH_DATAGRAMS_TURN, SP_PUBLISH_BYTES_TURN,
sp_ns_now_ns()))
return;
}
if ((unsigned int)received < requested)
return;
}
}
/* ==================================================================
* Core-driven TCP adapter (direct/SOCKS/gateway/publish lifecycle)
*
* This is the sole live TCP path. A direct-TCP connection's handshake runs
* through the pure sp_tcp_core: child SYN -> FLOW_ADMITTED (passive) -> OPEN_STREAM ->
* real backend connect() -> connect result -> SYN-ACK emitted through the scheduler ->
* child ACK -> ESTABLISHED. The egress/ingress executors execute the
* bidirectional data effects and the scheduler schedules child-facing
* control/data. SOCKS uses the same ownership; the
* published-listener adapter delegates child segments to its bounded publish
* table before ordinary tuple admission.
* ================================================================== */
struct sp_ns_cold_flow {
char socks_target_domain[256];
struct socks_io socks;
uint8_t socks_residual[sizeof(((struct socks_io *)0)->rxbuf)];
size_t socks_residual_len;
};
static struct sp_ns_cold_flow g_ns_cold_flows[MAX_TCP];
struct sp_ns_flow {
uint8_t in_use;
uint8_t established;
uint8_t has_pending_emit; /* a TUN emit was FIFO-blocked and awaits re-drive */
uint8_t has_sched_emit_result; /* executor result waits for scheduler service */
uint8_t sched_emit_written;
uint8_t sched_restart_software;
uint8_t egress_force_software;
uint8_t waits_sched_credit;
uint8_t use_socks;
uint8_t socks_residual_active;
uint8_t admission_role;
uint8_t admission_source_held;
uint8_t backend_epoll_registered;
struct epoll_event backend_interest;
uint8_t backend_read_idle; /* last observe made no progress; suppress EPOLLIN
* while flight is unconsumed to avoid a 100% CPU
* spin when the child stalls (dup-ACKs, no ACK
* advance). Cleared by a child ACK that consumes. */
uint32_t generation; /* epoll token generation, nonzero when live */
uint32_t admission_source_ip;
int backend_fd;
struct sp_ns_cold_flow *cold;
struct sp_tcp_core local_core;
int pump_close_graceful;
uint8_t challenge_ack;
struct sp_egress_executor_flow egress; /* backend -> child */
struct sp_ingress_flow ingress; /* child -> backend */
/* The exact TCP_EMIT_BATCH effect retained when the TUN output FIFO
* was full. The core holds its matching tun_emit_effect and will neither
* re-emit nor retransmit until this effect's result arrives, so the adapter
* must re-execute *this* effect on TUN writability rather than re-derive. */
struct sp_fx_tcp_emit_batch pending_emit;
struct sp_ev_tun_emit_result sched_emit_result;
uint64_t sched_next_seq;
uint64_t sched_last_data_seq;
uint64_t sched_terminal_seq;
unsigned sched_emit_outstanding;
unsigned sched_blocked_dims;
struct sp_scheduler_credit sched_blocked_credit;
uint64_t flow_id;
uint64_t core_gen;
uint64_t last_active_ns;
uint64_t connect_effect_id; /* OPEN_STREAM effect awaiting its result */
/* Child-facing addressing, network byte order as read from the TUN packet. */
uint32_t cli_ip;
uint32_t srv_ip;
uint16_t cli_port;
uint16_t srv_port;
struct sp_dns_ref dns_mapping_ref;
};
static struct sp_ns_flow g_ns_flows[MAX_TCP];
static uint64_t g_ns_output_work[(MAX_TCP + 63) / 64];
static void sp_ns_output_work_refresh(const struct sp_ns_flow *f) {
size_t slot = (size_t)(f - g_ns_flows);
uint64_t bit = UINT64_C(1) << (slot % 64);
if (f->in_use && (f->has_pending_emit || f->has_sched_emit_result))
g_ns_output_work[slot / 64] |= bit;
else
g_ns_output_work[slot / 64] &= ~bit;
}
/* Find the next occupied output-work slot without touching idle flow state. */
static size_t sp_ns_output_work_next(size_t start) {
size_t limit = sp_ns_tcp_slot_limit();
while (start < limit) {
uint64_t bits = g_ns_output_work[start / 64] &
(UINT64_MAX << (start % 64));
if (bits) {
size_t slot = (start / 64) * 64 + (size_t)__builtin_ctzll(bits);
return slot < limit ? slot : limit;
}
start = (start / 64 + 1) * 64;
}
return limit;
}
static uint16_t g_ns_free_next[MAX_TCP];
static uint16_t g_ns_free_head;
static void sp_ns_slots_init(void) {
memset(g_ns_output_work, 0, sizeof(g_ns_output_work));
size_t count = sp_ns_tcp_slot_limit();
g_ns_free_head = count ? 0 : UINT16_MAX;
for (size_t i = 0; i < count; ++i)
g_ns_free_next[i] = i + 1 < count ? (uint16_t)(i + 1) : UINT16_MAX;
}
/* storage must be freshly zero-filled (sp_maps_startup() maps it
* MAP_ANONYMOUS): the entry arrays are left untouched, so their pages are only
* faulted in when first used. */
static int sp_maps_init(struct sp_map_storage *storage,
const struct ht_hash_key secrets[4]) {
g_tcp_map = (struct sp_tcp_map){.secret = secrets[0], .flows = g_ns_flows};
g_udp_map = (struct sp_udp_map){.secret = secrets[1],
.entries = storage->udp_entries, .free_slots = storage->udp_free,
.free_count = SP_UDP_KEY_MAX};
for (uint32_t i = 0; i < SP_UDP_KEY_MAX; ++i)
storage->udp_free[i] = SP_UDP_KEY_MAX - 1U - i;
g_timewait_map = (struct sp_timewait_map){.secret = secrets[2],
.entries = storage->timewait_entries, .free_slots = storage->timewait_free,
.free_count = SP_TIMEWAIT_ALL_MAX};
for (uint32_t i = 0; i < SP_TIMEWAIT_ALL_MAX; ++i)
storage->timewait_free[i] = SP_TIMEWAIT_ALL_MAX - 1U - i;
g_admission_map = (struct sp_admission_map){.secret = secrets[3],
.entries = storage->admission_entries, .free_slots = storage->admission_free,
.free_count = SP_ADMISSION_KEY_MAX};
for (uint32_t i = 0; i < SP_ADMISSION_KEY_MAX; ++i)
storage->admission_free[i] = SP_ADMISSION_KEY_MAX - 1U - i;
g_ns_timewait_heap_count = 0;
return ht_init(&g_tcp_map.index, storage->tcp, SP_TCP_TABLE_SLOTS, MAX_TCP, MAX_TCP) &&
ht_init(&g_udp_map.index, storage->udp, SP_UDP_TABLE_SLOTS, SP_UDP_KEY_MAX, SP_UDP_KEY_MAX) &&
ht_init(&g_timewait_map.index, storage->timewait, SP_TIMEWAIT_TABLE_SLOTS, SP_TIMEWAIT_ALL_MAX, SP_TIMEWAIT_ALL_MAX) &&
ht_init(&g_admission_map.index, storage->admission, SP_ADMISSION_TABLE_SLOTS, SP_ADMISSION_KEY_MAX, SP_ADMISSION_KEY_MAX) ? 0 : -1;
}
static int sp_maps_startup(void) {
struct ht_hash_key secrets[4];
for (size_t i = 0; i < 4; ++i)
if (!ht_hash_key_random(&secrets[i])) return -1;
g_map_storage = mmap(NULL, sizeof(*g_map_storage), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (g_map_storage == MAP_FAILED) { g_map_storage = NULL; return -1; }
if (sp_maps_init(g_map_storage, secrets) < 0) {
munmap(g_map_storage, sizeof(*g_map_storage));
g_map_storage = NULL;
return -1;
}
return 0;
}
static void sp_ns_timer_refresh(struct sp_ns_flow *f) {
unsigned owner = (unsigned)(f - g_ns_flows);
uint64_t deadline = sp_tcp_next_deadline_ns(&f->local_core, NULL);
if (!deadline && f->in_use)
deadline = sp_udp_deadline(f->last_active_ns,
(unsigned)(f->established ? TCP_IDLE_TIMEOUT_SEC : TCP_HALF_OPEN_TIMEOUT_SEC));
sp_deadline_set(&g_net_deadlines, owner, f->in_use ? deadline : 0);
}
static uint64_t g_ns_next_flow_id = 1;
static uint32_t g_ns_epoll_generation = 1;
/* Lets the effect executor reach the TUN writer without threading tunfd through
* every effect. Set for the duration of one dispatched packet or backend
* event. */
static struct sp_scheduler_sched g_ns_tun_sched;
static struct sp_scheduler_overload g_ns_overload;
/* Explicit authority for precisely one synchronous TCP_EMIT_BATCH callback. */
struct sp_ns_emit_transaction {
struct sp_ns_flow *flow;
const struct sp_fx_tcp_emit_batch *effect;
int tunfd;
};
static int g_ns_sched_servicing;
/* Process-local secret for scheduler-only auxiliary tuple identities. These
* ids are never TCP flow ids and are used only by the bounded per-source
* accounting/coalescing tables, so a child must not be able to precompute
* collisions that evict another tuple's bucket state. */
static void sp_resource_print_baseline(const char *point) {
uint64_t admission_refs = 0;
size_t tcp_flows = 0;
for (size_t i = 0; i < sp_ns_tcp_slot_limit(); ++i)
if (g_ns_flows[i].in_use)
tcp_flows++;
for (size_t i = 0; i < SP_ADMISSION_KEY_MAX; ++i) {
const struct sp_admission_entry *entry =
&g_admission_map.entries[i];
if (entry->live)
admission_refs += entry->refs;
}
(void)sp_stderr_printf(
"sp_runtime_baseline\t%s\tpid\t%ld\t"
"tcp_flows_current\t%zu\ttimewait_current\t%u\t"
"admission_refs_current\t%llu\treservation_current\t%llu\t"
"scheduler_control_current\t%zu\tscheduler_data_current\t%zu\t"
"tun_queue_current\t%zu\ttun_queue_high_water\t%llu\n",
point, (long)g_runtime_role_pid, tcp_flows,
g_ns_timewait_heap_count,
(unsigned long long)admission_refs,
(unsigned long long)g_resource_counters.reservation_current,
g_ns_tun_sched.control.count, g_ns_tun_sched.data.count,
g_tun_queue.count,
(unsigned long long)g_resource_counters.tun_queue_high_water);
}
/* Scheduler-owned responses without a live TCP flow (CLOSED RSTs and ICMP
* errors for UDP) need a stable identity for the overload policy's buckets/coalescing, but must
* never be mistaken for a reusable live-flow id. */
#define SP_NS_AUX_FLOW_TAG (UINT64_C(1) << 63)
static uint64_t sp_ns_aux_flow_id(uint32_t cli_ip, uint32_t srv_ip,
uint16_t cli_port, uint16_t srv_port,
uint8_t protocol) {
uint64_t hash = g_ns_aux_hash_key ^ UINT64_C(1469598103934665603);
const uint8_t bytes[] = {
(uint8_t)(cli_ip >> 24), (uint8_t)(cli_ip >> 16),
(uint8_t)(cli_ip >> 8), (uint8_t)cli_ip,
(uint8_t)(srv_ip >> 24), (uint8_t)(srv_ip >> 16),
(uint8_t)(srv_ip >> 8), (uint8_t)srv_ip,
(uint8_t)(cli_port >> 8), (uint8_t)cli_port,
(uint8_t)(srv_port >> 8), (uint8_t)srv_port, protocol,
};
size_t i;
for (i = 0; i < sizeof(bytes); ++i) {
hash ^= bytes[i];
hash *= UINT64_C(1099511628211);
}
return SP_NS_AUX_FLOW_TAG | (hash & ~SP_NS_AUX_FLOW_TAG);
}
static int sp_ns_is_aux_flow(uint64_t flow) {
return (flow & SP_NS_AUX_FLOW_TAG) != 0;
}
static uint64_t sp_ns_timewait_hash(uint32_t cli_ip, uint32_t srv_ip,
uint16_t cli_port, uint16_t srv_port) {
return sp_ns_aux_flow_id(ntohl(cli_ip), ntohl(srv_ip), cli_port, srv_port,
IPPROTO_TCP);
}
static int sp_ns_timewait_key_equal(const struct sp_ns_timewait_entry *entry,
uint32_t cli_ip, uint32_t srv_ip,
uint16_t cli_port, uint16_t srv_port) {
return entry->cli_ip == cli_ip && entry->srv_ip == srv_ip &&
entry->cli_port == cli_port && entry->srv_port == srv_port;
}
static void sp_resource_admission_release_source(uint32_t cli_ip, uint8_t role);
static void sp_ns_timewait_heap_add(struct sp_ns_timewait_entry *entry) {
unsigned owner = SP_DEADLINE_TIMEWAIT + (unsigned)(entry - g_timewait_map.entries);
if (!g_net_deadlines.positions[owner]) ++g_ns_timewait_heap_count;
sp_deadline_set(&g_net_deadlines, owner, entry->expiry_ns);
}
static void sp_ns_timewait_heap_detach(struct sp_ns_timewait_entry *entry) {
unsigned owner = SP_DEADLINE_TIMEWAIT + (unsigned)(entry - g_timewait_map.entries);
if (g_net_deadlines.positions[owner]) --g_ns_timewait_heap_count;
sp_deadline_set(&g_net_deadlines, owner, 0);
}
static void sp_ns_timewait_heap_refresh(struct sp_ns_timewait_entry *entry,
uint64_t expiry_ns) {
if (entry == NULL || entry->state != SP_NS_TIMEWAIT_LIVE)
return;
sp_ns_timewait_heap_detach(entry);
entry->expiry_ns = expiry_ns;
sp_ns_timewait_heap_add(entry);
}
static enum ht_step sp_timewait_map_probe(const struct sp_timewait_map *m,
const struct sp_timewait_key *key, struct ht_probe *p, uint32_t *slot,
uint32_t *bucket) {
*p = ht_probe_begin(&m->index, ht_siphash13(key, sizeof(*key), &m->secret));
enum ht_step step;
while ((step = ht_probe_next(&m->index, p, slot, bucket)) == HT_CANDIDATE)
if (sp_ns_timewait_key_equal(&m->entries[*slot], key->cli_ip, key->srv_ip,
key->cli_port, key->srv_port)) break;
if (step == HT_CORRUPT || step == HT_EXHAUSTED) abort();
return step;
}
static struct sp_ns_timewait_entry *
sp_ns_timewait_lookup(uint32_t cli_ip, uint32_t srv_ip, uint16_t cli_port,
uint16_t srv_port) {
const struct sp_timewait_key key = {cli_ip, srv_ip, cli_port, srv_port};
struct ht_probe p;
uint32_t slot, bucket;
return sp_timewait_map_probe(&g_timewait_map, &key, &p, &slot, &bucket) == HT_CANDIDATE
? &g_timewait_map.entries[slot] : NULL;
}
static int sp_ns_timewait_insert(uint32_t cli_ip, uint32_t srv_ip,
uint16_t cli_port, uint16_t srv_port,
uint64_t snd_nxt_abs, uint64_t rcv_acked_abs,
uint64_t rcv_adv_right_abs, uint64_t expiry_ns,
uint8_t role,
uint32_t admission_source_ip) {
struct sp_timewait_map *m = &g_timewait_map;
const struct sp_timewait_key key = {cli_ip, srv_ip, cli_port, srv_port};
struct ht_probe p;
uint32_t slot, bucket;
bool exists = sp_timewait_map_probe(m, &key, &p, &slot, &bucket) == HT_CANDIDATE;
if (!exists) {
if (!m->free_count) return -1;
slot = m->free_slots[m->free_count - 1];
}
struct sp_ns_timewait_entry *entry = &m->entries[slot];
uint16_t flags = exists ? entry->reserved : 0;
/* A refresh cannot transfer an admission reference to a different source. */
if (exists && (flags & SP_NS_TIMEWAIT_ADMISSION_REF) &&
(entry->role != role || entry->admission_source_ip != admission_source_ip))
return -1;
*entry = (struct sp_ns_timewait_entry){
.cli_ip = cli_ip, .srv_ip = srv_ip, .cli_port = cli_port, .srv_port = srv_port,
.snd_nxt_abs = snd_nxt_abs, .rcv_acked_abs = rcv_acked_abs,
.rcv_adv_right_abs = rcv_adv_right_abs, .expiry_ns = expiry_ns,
.state = SP_NS_TIMEWAIT_LIVE, .role = role, .reserved = flags,
.admission_source_ip = admission_source_ip};
if (!exists) {
if (ht_insert_at(&m->index, &p, bucket, slot) != HT_INSERTED) abort();
--m->free_count;
}
sp_ns_timewait_heap_add(entry);
return 0;
}
static void sp_ns_timewait_remove(struct sp_ns_timewait_entry *entry) {
if (entry == NULL || entry->state != SP_NS_TIMEWAIT_LIVE) return;
struct sp_timewait_map *m = &g_timewait_map;
const struct sp_timewait_key key = {entry->cli_ip, entry->srv_ip, entry->cli_port, entry->srv_port};
struct ht_probe p;
uint32_t slot, bucket;
if (sp_timewait_map_probe(m, &key, &p, &slot, &bucket) != HT_CANDIDATE ||
&m->entries[slot] != entry || ht_erase_at(&m->index, bucket) != slot) abort();
sp_ns_timewait_heap_detach(entry);
if (entry->reserved & SP_NS_TIMEWAIT_ADMISSION_REF)
sp_resource_admission_release_source(entry->admission_source_ip, entry->role);
entry->state = SP_NS_TIMEWAIT_EMPTY;
m->free_slots[m->free_count++] = slot;
}
static void sp_resource_admission_limits(const struct sp_admission_key *key, uint64_t *rate,
uint64_t *burst) {
if (key->bytes[4] == SP_UDP_ADMISSION_GLOBAL) {
*rate = SP_UDP_ADMIT_ALL_PER_S;
*burst = SP_UDP_ADMIT_ALL_BURST;
} else if (key->bytes[4] == SP_UDP_ADMISSION_RULE) {
*rate = SP_UDP_ADMIT_RULE_S;
*burst = SP_UDP_ADMIT_RULE_B;
} else if ((key->bytes[4] & 0x80U) != 0) {
*rate = SP_UDP_ADMIT_SOURCE_S;
*burst = SP_UDP_ADMIT_SOURCE_B;
} else {
*rate = MAX_CONNECTS_PER_SEC;
*burst = MAX_CONNECTS_PER_SEC;
}
}
static void sp_resource_admission_refill(struct sp_resource_admission_value *value,
uint64_t now_ns, uint64_t rate,
uint64_t burst) {
uint64_t max_allowance = burst * SP_SCHEDULER_TOKEN_NS;
uint64_t elapsed;
uint64_t add;
if (now_ns <= value->last_ns) {
value->last_ns = now_ns;
return;
}
elapsed = now_ns - value->last_ns;
value->last_ns = now_ns;
if (elapsed > max_allowance / rate + 1)
elapsed = max_allowance / rate + 1;
add = elapsed * rate;
if (max_allowance - value->allowance <= add)
value->allowance = max_allowance;
else
value->allowance += add;
}
static int sp_resource_admission_take(struct sp_admission_entry *entry,
uint64_t now_ns) {
uint64_t rate, burst;
if (entry == NULL || !entry->live) return 0;
sp_resource_admission_limits(&entry->key, &rate, &burst);
sp_resource_admission_refill(&entry->value, now_ns, rate, burst);
if (entry->value.allowance < SP_SCHEDULER_TOKEN_NS) return 0;
entry->value.allowance -= SP_SCHEDULER_TOKEN_NS;
return 1;
}
static enum ht_step sp_admission_map_probe(const struct sp_admission_map *m,
const struct sp_admission_key *key, struct ht_probe *p, uint32_t *slot,
uint32_t *bucket) {
*p = ht_probe_begin(&m->index, ht_siphash13(key->bytes, sizeof(key->bytes), &m->secret));
enum ht_step step;
while ((step = ht_probe_next(&m->index, p, slot, bucket)) == HT_CANDIDATE)
if (memcmp(m->entries[*slot].key.bytes, key->bytes, sizeof(key->bytes)) == 0) break;
if (step == HT_CORRUPT || step == HT_EXHAUSTED) abort();
return step;
}
static struct sp_admission_entry *sp_resource_admission_lookup(const struct sp_admission_key *key) {
struct ht_probe p;
uint32_t slot, bucket;
return sp_admission_map_probe(&g_admission_map, key, &p, &slot, &bucket) == HT_CANDIDATE
? &g_admission_map.entries[slot] : NULL;
}
static void sp_admission_reclaim(struct sp_admission_entry *entry) {
struct sp_admission_map *m = &g_admission_map;
struct ht_probe p;
uint32_t slot, bucket;
if (sp_admission_map_probe(m, &entry->key, &p, &slot, &bucket) != HT_CANDIDATE ||
&m->entries[slot] != entry || ht_erase_at(&m->index, bucket) != slot) abort();
entry->live = false;
m->free_slots[m->free_count++] = slot;
}
static struct sp_admission_entry *sp_resource_admission_acquire(const struct sp_admission_key *key) {
struct sp_admission_map *m = &g_admission_map;
struct ht_probe p;
uint32_t slot, bucket;
struct sp_admission_entry *entry;
if (sp_admission_map_probe(m, key, &p, &slot, &bucket) == HT_CANDIDATE) {
entry = &m->entries[slot];
if (entry->refs == UINT64_MAX) return NULL;
++entry->refs;
return entry;
}
uint64_t now_ns = sp_ns_now_ns(), rate, burst;
/* Reclaim only under pool pressure, before starting the insertion probe.
* Depleted zero-owner buckets retain their tokens: reconnect cannot reset a
* source limit. This bounded scan is at most 512 cold records. */
if (!m->free_count) {
for (uint32_t i = 0; i < SP_ADMISSION_KEY_MAX; ++i) {
entry = &m->entries[i];
if (!entry->live || entry->refs) continue;
sp_resource_admission_limits(&entry->key, &rate, &burst);
sp_resource_admission_refill(&entry->value, now_ns, rate, burst);
if (entry->value.allowance == burst * SP_SCHEDULER_TOKEN_NS) {
sp_admission_reclaim(entry);
break;
}
}
if (!m->free_count) return NULL;
/* Backshift deletion invalidated the original insertion position. */
if (sp_admission_map_probe(m, key, &p, &slot, &bucket) != HT_VACANT) abort();
}
slot = m->free_slots[m->free_count - 1];
entry = &m->entries[slot];
sp_resource_admission_limits(key, &rate, &burst);
*entry = (struct sp_admission_entry){.key = *key, .live = true, .refs = 1,
.value = {.allowance = burst * SP_SCHEDULER_TOKEN_NS, .last_ns = now_ns}};
if (ht_insert_at(&m->index, &p, bucket, slot) != HT_INSERTED) abort();
--m->free_count;
return entry;
}
static void sp_resource_admission_release(struct sp_admission_entry *entry) {
if (entry == NULL) return;
if (!entry->live || !entry->refs) abort();
if (--entry->refs) return;
uint64_t rate, burst;
sp_resource_admission_limits(&entry->key, &rate, &burst);
sp_resource_admission_refill(&entry->value, sp_ns_now_ns(), rate, burst);
if (entry->value.allowance == burst * SP_SCHEDULER_TOKEN_NS)
sp_admission_reclaim(entry);
}
static void sp_resource_admission_source_key(struct sp_admission_key *key,
uint32_t cli_ip, uint8_t role) {
memcpy(key->bytes, &cli_ip, sizeof(cli_ip));
key->bytes[4] = role;
}
static void sp_resource_admission_release_source(uint32_t cli_ip, uint8_t role) {
struct sp_admission_key key;
sp_resource_admission_source_key(&key, cli_ip, role);
sp_resource_admission_release(sp_resource_admission_lookup(&key));
}
/* One global child-to-backend staging pool (design: bounded global pool; the
* per-flow K_rx cap is enforced by ingress.staged) and its arena. */
static uint8_t g_ns_stage_arena[(size_t)SP_INGRESS_STAGE_SLOTS * SP_TUN_FRAME_MAX_BYTES];
static struct sp_ingress_stage_pool g_ns_stage_pool;
/* A reservation slot is assigned only while a flow owns retained child bytes.
* The global pool is one eighth of the former 128 x 256 KiB matrix. A flow is
* assigned at most one slot and only while it owns retained bytes, so one
* blocked flow cannot consume the pool. Published flows can acquire only the
* high eight slots; child-originated direct and SOCKS flows prefer the low
* eight and can acquire all sixteen. Slot pages remain demand-resident. */
static uint8_t
g_ns_reservation[SP_INGRESS_RESERVATION_SLOTS][SP_INGRESS_RESERVATION_SLOT_BYTES];
static uint64_t g_ns_reservation_owner[SP_INGRESS_RESERVATION_SLOTS];
static uint64_t g_ns_reservation_generation[SP_INGRESS_RESERVATION_SLOTS];
static uint8_t *sp_ns_reservation_acquire(struct sp_ingress_flow *flow,
uint8_t role) {
size_t i;
size_t first = role == SP_TCP_ROLE_PUBLISH
? SP_INGRESS_RESERVATION_SLOTS -
SP_INGRESS_PUBLISH_RESERVATION_SLOTS
: 0;
if (flow->reservation_slot != UINT16_MAX) {
i = flow->reservation_slot;
if (i < SP_INGRESS_RESERVATION_SLOTS &&
g_ns_reservation_owner[i] == flow->flow &&
g_ns_reservation_generation[i] == flow->generation)
return g_ns_reservation[i];
return NULL;
}
for (i = first; i < SP_INGRESS_RESERVATION_SLOTS; ++i) {
if (g_ns_reservation_owner[i] == 0) {
g_ns_reservation_owner[i] = flow->flow;
g_ns_reservation_generation[i] = flow->generation;
flow->reservation_slot = (uint16_t)i;
sp_counter_inc(&g_resource_counters.reservation_current);
if (g_resource_metrics_enabled &&
g_resource_counters.reservation_current >
g_resource_counters.reservation_high_water)
g_resource_counters.reservation_high_water =
g_resource_counters.reservation_current;
return g_ns_reservation[i];
}
}
sp_counter_inc(&g_resource_counters.reservation_acquire_failures);
return NULL;
}
static void sp_ns_reservation_release(struct sp_ingress_flow *flow) {
size_t slot;
if (flow->reservation_slot == UINT16_MAX) {
if (flow->pending_bytes != 0)
abort();
return;
}
slot = flow->reservation_slot;
if (flow->flow == 0 || flow->generation == 0 ||
slot >= SP_INGRESS_RESERVATION_SLOTS ||
g_ns_reservation_owner[slot] != flow->flow ||
g_ns_reservation_generation[slot] != flow->generation) {
if (flow->pending_bytes != 0)
abort();
flow->reservation_slot = UINT16_MAX;
return;
}
if (flow->pending_bytes > g_ns_stage_pool.global_reserved)
abort();
g_ns_stage_pool.global_reserved -= flow->pending_bytes;
flow->pending_bytes = 0;
flow->reserved_base = 0;
g_ns_reservation_owner[slot] = 0;
g_ns_reservation_generation[slot] = 0;
if (g_resource_metrics_enabled && g_resource_counters.reservation_current > 0)
g_resource_counters.reservation_current--;
flow->reservation_slot = UINT16_MAX;
}
/* Shared non-reentrant scratch for the egress peek/materialize/consume paths. */
static uint8_t g_ns_discard[SP_TCP_FLIGHT_PER_FLOW];
static uint8_t g_ns_payload[(size_t)SP_TCP_CORE_MAX_SEGMENTS * SP_TCP_CORE_MAX_MSS];
/* The egress write callback admits the exact framed batch to the scheduler. The executor result
* is held until scheduler service reaches the final TUN sink, so core send
* state never advances merely because a record was copied into a scheduler arena. */
static ssize_t sp_ns_tun_write(void *context, const uint8_t *record,
size_t len) {
struct sp_ns_emit_transaction *txn = context;
struct sp_ns_flow *flow;
const struct sp_fx_tcp_emit_batch *effect;
struct sp_scheduler_record queued;
unsigned blocked_dims;
size_t flow_index;
if (txn == NULL || txn->tunfd < 0 || txn->flow == NULL ||
txn->effect == NULL)
return -1;
flow = txn->flow;
effect = txn->effect;
flow_index = (size_t)(flow - g_ns_flows);
if (flow_index >= sp_ns_tcp_slot_limit() || flow_index > UINT16_MAX ||
!flow->in_use || flow->flow_id == 0 || flow->core_gen == 0)
return -1;
memset(&queued, 0, sizeof(queued));
queued.flow = flow->flow_id;
queued.generation = flow->core_gen;
queued.klass = SP_SCHEDULER_CLASS_DATA;
queued.tier = SP_SCHEDULER_CTRL_ORDINARY;
queued.seq_abs = flow->sched_next_seq + 1;
if (len <= SP_VNET_HDR_BYTES + 1500U) {
queued.logical_bytes = len > SP_VNET_HDR_BYTES + 40U
? len - SP_VNET_HDR_BYTES - 40U
: len;
queued.logical_packets = 1;
} else {
queued.logical_bytes = effect->end.value - effect->start.value;
queued.logical_packets =
(unsigned)((queued.logical_bytes + effect->mss - 1) / effect->mss);
}
if (sp_scheduler_admit_with_flow_index(
&g_ns_tun_sched, &queued, record, len, &blocked_dims,
(uint16_t)flow_index) < 0) {
if (blocked_dims == 0) {
errno = EINVAL;
return -1;
}
flow->sched_blocked_credit = sp_scheduler_credit_now(&g_ns_tun_sched);
flow->sched_blocked_dims = blocked_dims;
flow->waits_sched_credit = 1;
errno = EAGAIN;
return -1;
}
flow->waits_sched_credit = 0;
flow->sched_blocked_dims = 0;
flow->sched_next_seq = queued.seq_abs;
flow->sched_last_data_seq = queued.seq_abs;
flow->pending_emit = *effect;
flow->sched_emit_outstanding++;
g_ns_sched_tun_pending = 1;
tun_update_events(txn->tunfd);
return (ssize_t)len;
}
static uint64_t sp_ns_now_ns(void) {
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0)
return 0;
return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec;
}
static uint64_t sp_ns_scheduler_clock(void *context) {
(void)context;
return sp_ns_now_ns();
}
static enum ht_step sp_tcp_map_probe(const struct sp_tcp_map *m,
const struct sp_tcp_key *key, struct ht_probe *p, uint32_t *slot, uint32_t *bucket) {
*p = ht_probe_begin(&m->index, ht_siphash13(key, sizeof(*key), &m->secret));
enum ht_step step;
while ((step = ht_probe_next(&m->index, p, slot, bucket)) == HT_CANDIDATE) {
const struct sp_ns_flow *f = &m->flows[*slot];
if (f->cli_ip == key->cli_ip && f->srv_ip == key->srv_ip &&
f->cli_port == key->cli_port && f->srv_port == key->srv_port) break;
}
if (step == HT_CORRUPT || step == HT_EXHAUSTED) abort();
return step;
}
static int sp_ns_tcp_table_insert(struct sp_ns_flow *f) {
const struct sp_tcp_key key = {f->cli_ip, f->srv_ip, f->cli_port, f->srv_port};
struct ht_probe p;
uint32_t slot, bucket;
if (sp_tcp_map_probe(&g_tcp_map, &key, &p, &slot, &bucket) == HT_CANDIDATE)
return -1;
return ht_insert_at(&g_tcp_map.index, &p, bucket, (uint32_t)(f - g_tcp_map.flows)) == HT_INSERTED ? 0 : -1;
}
static void sp_ns_tcp_table_remove(const struct sp_ns_flow *f) {
const struct sp_tcp_key key = {f->cli_ip, f->srv_ip, f->cli_port, f->srv_port};
struct ht_probe p;
uint32_t slot, bucket;
if (sp_tcp_map_probe(&g_tcp_map, &key, &p, &slot, &bucket) == HT_CANDIDATE &&
&g_tcp_map.flows[slot] == f)
if (ht_erase_at(&g_tcp_map.index, bucket) != slot) abort();
}
static struct sp_ns_flow *sp_ns_find(uint32_t cli_ip, uint16_t cli_port,
uint32_t srv_ip, uint16_t srv_port) {
const struct sp_tcp_key key = {cli_ip, srv_ip, cli_port, srv_port};
struct ht_probe p;
uint32_t slot, bucket;
return sp_tcp_map_probe(&g_tcp_map, &key, &p, &slot, &bucket) == HT_CANDIDATE
? &g_tcp_map.flows[slot] : NULL;
}
static struct sp_ns_timewait_entry *
sp_ns_timewait_find_live(uint32_t cli_ip, uint32_t srv_ip, uint16_t cli_port,
uint16_t srv_port, uint64_t now_ns) {
struct sp_ns_timewait_entry *entry =
sp_ns_timewait_lookup(cli_ip, srv_ip, cli_port, srv_port);
if (entry != NULL && entry->expiry_ns <= now_ns) {
sp_ns_timewait_remove(entry);
entry = NULL;
}
return entry;
}
static struct sp_ns_flow *sp_ns_alloc(void) {
uint16_t slot = g_ns_free_head;
if (slot == UINT16_MAX) return NULL;
g_ns_free_head = g_ns_free_next[slot];
return &g_ns_flows[slot];
}
static int sp_ns_tcp_admission_capacity(uint8_t role) {
size_t live = 0;
size_t role_live = 0;
size_t half_open = 0;
size_t role_half_open = 0;
size_t connecting = 0;
for (size_t i = 0; i < sp_ns_tcp_slot_limit(); ++i) {
const struct sp_ns_flow *flow = &g_ns_flows[i];
if (!flow->in_use)
continue;
live++;
if (flow->admission_role == role)
role_live++;
if (flow->local_core.state < SP_TCP_CORE_ESTABLISHED) {
half_open++;
if (flow->admission_role == role)
role_half_open++;
}
if (!flow->established)
connecting++;
}
if (live >= sp_ns_tcp_slot_limit() || half_open >= SP_TCP_HALF_OUT_MAX ||
connecting >= SP_CONNECT_INFLIGHT_MAX)
return 0;
if (role == SP_TCP_ROLE_DIRECT)
return role_live < SP_TCP_DIRECT_MAX &&
role_half_open < SP_TCP_HALF_DIRECT_MAX;
if (role == SP_TCP_ROLE_SOCKS)
return role_live < SP_TCP_SOCKS_MAX &&
role_half_open < SP_TCP_HALF_SOCKS_MAX;
return 0;
}
static int sp_ns_pump_start(struct sp_ns_flow *f, struct sp_pump_item initial,
uint64_t staged_id, int insert_on_complete);
static void sp_ns_flow_release(struct sp_ns_flow *f) {
unsigned slot = (unsigned)(f - g_ns_flows);
sp_deadline_set(&g_net_deadlines, slot, 0);
if (f->in_use) {
g_ns_free_next[slot] = g_ns_free_head;
g_ns_free_head = (uint16_t)slot;
}
sp_scheduler_retire_flow(&g_ns_tun_sched, f->flow_id, f->core_gen);
sp_scheduler_overload_retire_flow(&g_ns_overload, f->flow_id);
/* Dispose any child frames still staged for this flow before the generation
* is cleared, or they leak the global staging pool. */
(void)sp_ingress_stage_cancel_flow(&g_ns_stage_pool, &f->ingress);
sp_ns_reservation_release(&f->ingress);
if (f->in_use)
sp_ns_tcp_table_remove(f);
if (f->backend_fd >= 0) {
if (f->backend_epoll_registered)
epoll_del(f->backend_fd);
close(f->backend_fd);
}
if (f->in_use && f->admission_source_held)
sp_resource_admission_release_source(f->admission_source_ip,
f->admission_role);
if (f->dns_mapping_ref.generation != 0)
(void)sp_dns_core_release(&g_dns_core, f->dns_mapping_ref,
sp_ns_now_ns());
memset(f, 0, sizeof(*f));
f->backend_fd = -1;
sp_ns_output_work_refresh(f);
}
static void sp_ns_release_all_flows(void) {
for (size_t i = 0; i < sp_ns_tcp_slot_limit(); ++i)
if (g_ns_flows[i].in_use)
sp_ns_flow_release(&g_ns_flows[i]);
}
/* Move a fully closed active-side core into the no-fd TIME-WAIT table only
* after its final control effects have reached the scheduler sink. If the
* bounded table cannot admit this tuple, retain the live flow: capacity pressure
* may reduce new admissions but must never weaken tuple quarantine. */
static int sp_ns_park_timewait(struct sp_ns_flow *f) {
if (f == NULL || !f->in_use || f->local_core.state != SP_TCP_CORE_TIME_WAIT ||
f->local_core.timewait_deadline_ns == 0)
return 0;
if (f->has_pending_emit || f->has_sched_emit_result ||
f->sched_emit_outstanding != 0)
return 0;
if (sp_ns_timewait_insert(
f->cli_ip, f->srv_ip, f->cli_port, f->srv_port,
f->local_core.snd_nxt_high_abs, f->local_core.rcv_acked_abs,
f->local_core.rcv_adv_right_abs, f->local_core.timewait_deadline_ns,
f->admission_role,
f->admission_source_ip) < 0)
return -1;
{
struct sp_admission_key key;
sp_resource_admission_source_key(&key, f->admission_source_ip,
f->admission_role);
if (sp_resource_admission_acquire(&key) == NULL) {
sp_ns_timewait_remove(
sp_ns_timewait_lookup(f->cli_ip, f->srv_ip, f->cli_port,
f->srv_port));
return -1;
}
{
struct sp_ns_timewait_entry *entry =
sp_ns_timewait_lookup(f->cli_ip, f->srv_ip, f->cli_port,
f->srv_port);
if (entry == NULL) {
sp_resource_admission_release_source(f->admission_source_ip,
f->admission_role);
return -1;
}
entry->reserved |= SP_NS_TIMEWAIT_ADMISSION_REF;
}
}
sp_ns_flow_release(f);
return 1;
}
static void sp_ns_maybe_park_timewait(struct sp_ns_flow *f) {
if (f != NULL && f->in_use && f->local_core.state == SP_TCP_CORE_TIME_WAIT)
(void)sp_ns_park_timewait(f);
}
/* Records the capability of the last STREAM_SEND the pump authorized this call,
* so the segment path can free a staged frame the core did not authorize (a
* duplicate or out-of-window retransmit) rather than orphan its slot. */
/* Count of staged child frames freed because the core did not authorize them
* (duplicate/out-of-window retransmits); a nonzero value under load is evidence
* the K_rx orphan-disposal path is exercised and not leaking. */
static uint64_t g_ns_orphans_freed = 0;
/* Set by the pump when the CLOSE_FLOW it processed was a graceful mutual-FIN
* close (so the teardown releases quietly instead of resetting the child). */
static void sp_ns_reset_child(struct sp_ns_flow *f);
static void sp_ns_teardown(struct sp_ns_flow *f);
static void sp_ns_service_tun_scheduler(int tunfd);
static void sp_ns_retry_pending_emit(int tunfd, struct sp_ns_flow *f);
static int sp_ns_handle_timewait(int tunfd,
struct sp_ns_timewait_entry *entry,
uint32_t seq, uint8_t flags,
size_t payload_len, uint64_t now_ns);
static int sp_ns_admit_aux_response(enum sp_scheduler_response_kind kind,
uint64_t flow, int in_reply_to_rst,
const uint8_t *record, size_t record_len,
uint64_t now_ns) {
enum sp_scheduler_shed_decision decision;
decision = sp_scheduler_overload_response_with_dependency(
&g_ns_overload, &g_ns_tun_sched, kind, flow, 1, in_reply_to_rst, record,
record_len, 1, 0, now_ns);
if (decision == SP_SCHEDULER_SHED_FATAL)
return -1;
if (decision == SP_SCHEDULER_SHED_ADMIT || decision == SP_SCHEDULER_SHED_COALESCED) {
g_ns_sched_tun_pending = 1;
tun_update_events(g_ns_tunfd_for_emit);
}
return 0;
}
static int sp_ns_emit_child_record(struct sp_ns_flow *f,
const struct sp_fx_tcp_emit_control *effect,
int challenge_ack) {
uint8_t record[SP_VNET_HDR_BYTES + 40];
enum sp_scheduler_response_kind kind;
enum sp_scheduler_shed_decision decision;
uint64_t seq_abs = f->sched_next_seq + 1;
uint64_t depends_on_seq;
uint64_t now_ns;
size_t flow_index;
size_t record_len = sp_vnet_build_tcp_record(
record, sizeof(record), ntohl(f->srv_ip), ntohl(f->cli_ip), f->srv_port,
f->cli_port, (uint32_t)effect->seq.value, (uint32_t)effect->ack.value,
(uint8_t)effect->flags, NULL, 0, 0);
if (record_len == 0) {
DBG("failed to build child TCP control record");
return -1;
}
flow_index = (size_t)(f - g_ns_flows);
if (flow_index >= sp_ns_tcp_slot_limit() || flow_index > UINT16_MAX ||
!f->in_use || f->flow_id == 0 || f->core_gen == 0)
return -1;
if (effect->terminal &&
(f->has_sched_emit_result || f->sched_terminal_seq != 0 ||
f->egress.next_control_seq == UINT64_MAX)) {
DBG("terminal control completion already pending");
return -1;
}
kind = ((effect->flags &
(SP_TCP_CORE_SYN | SP_TCP_CORE_FIN | SP_TCP_CORE_RST)) != 0 ||
effect->terminal)
? SP_SCHEDULER_RESP_TERMINAL
: SP_SCHEDULER_RESP_CURRENT_ACK;
if (kind == SP_SCHEDULER_RESP_CURRENT_ACK && challenge_ack)
kind = SP_SCHEDULER_RESP_CHALLENGE_ACK;
depends_on_seq = kind == SP_SCHEDULER_RESP_TERMINAL ? f->sched_last_data_seq : 0;
now_ns = sp_ns_now_ns();
decision = sp_scheduler_overload_response_with_dependency_index(
&g_ns_overload, &g_ns_tun_sched, kind, f->flow_id, f->core_gen, 0,
record, record_len, seq_abs, depends_on_seq, now_ns,
(uint16_t)flow_index);
if (decision == SP_SCHEDULER_SHED_FATAL) {
DBG("terminal control scheduler admission failed");
return -1;
}
if (decision == SP_SCHEDULER_SHED_ADMIT || decision == SP_SCHEDULER_SHED_COALESCED) {
f->sched_next_seq = seq_abs;
/* A FIN effect occupies the core's one live TUN-emission slot just like a
* data batch. Hold its zero-byte success until the scheduler reaches the final TUN
* sink; otherwise the slot is never retired and every later RTO is
* suppressed behind the permanently-live FIN. SYN/RST/current-ACK
* controls do not own tun_emit_effect and therefore need no core result. */
if (effect->terminal) {
memset(&f->sched_emit_result, 0, sizeof(f->sched_emit_result));
f->sched_emit_result.context.key = sp_flow_key_make(
(sp_flow_id){.value = f->flow_id},
(sp_generation_id){.value = f->core_gen});
f->sched_emit_result.context.now.value = now_ns;
f->sched_emit_result.context.control_sequence.value =
++f->egress.next_control_seq;
f->sched_emit_result.effect = effect->context.effect;
f->sched_emit_result.outcome = SP_EMIT_ACCEPTED;
f->sched_terminal_seq = seq_abs;
f->has_sched_emit_result = 1;
sp_ns_output_work_refresh(f);
}
g_ns_sched_tun_pending = 1;
tun_update_events(g_ns_tunfd_for_emit);
}
return 0;
}
static int sp_ns_open_backend(struct sp_ns_flow *f,
const struct sp_fx_open_stream *effect) {
struct sockaddr_in addr;
int fd;
int rc;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
if (is_gateway_ip(f->srv_ip)) {
addr.sin_addr.s_addr = gateway_to_localhost(f->srv_ip);
addr.sin_port = htons(f->srv_port);
} else if (f->use_socks) {
addr = socks_proxy.addr;
} else {
addr.sin_addr.s_addr = f->srv_ip;
addr.sin_port = htons(f->srv_port);
}
fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
if (fd < 0)
return -1;
{
const char *option_reason = NULL;
if (sp_scheduler_configure_host_tcp_socket(fd, &option_reason) < 0) {
close(fd);
return -1;
}
}
rc = start_nonblocking_connect(fd, &addr);
if (rc < 0) {
close(fd);
return -1;
}
f->backend_fd = fd;
f->connect_effect_id = effect->context.effect.value;
if (f->use_socks) {
socks_io_reset(&f->cold->socks);
f->cold->socks.is_udp = 0;
f->cold->socks.target_ip = f->srv_ip;
f->cold->socks.target_port = f->srv_port;
snprintf(f->cold->socks.target_domain, sizeof(f->cold->socks.target_domain), "%s",
f->cold->socks_target_domain);
f->cold->socks.connect_pending = rc > 0;
f->cold->socks.state = f->cold->socks.connect_pending ? SOCKS_IO_CONNECTING
: SOCKS_IO_METHOD;
if (!f->cold->socks.connect_pending &&
socks_begin_handshake(&f->cold->socks, &socks_proxy) < 0) {
close(fd);
f->backend_fd = -1;
return -1;
}
}
f->generation = g_ns_epoll_generation;
g_ns_epoll_generation = epoll_next_generation(g_ns_epoll_generation);
{
size_t index = (size_t)(f - g_ns_flows);
struct epoll_event ev = {
.events = EPOLLOUT | EPOLLRDHUP | EPOLLERR | EPOLLHUP,
.data.u64 = epoll_registration_token(FD_TCP_CORE, index, fd,
f->generation)};
if (g_epfd >= 0) {
if (sp_epoll_update(EPOLL_CTL_ADD, fd, &ev, &f->backend_interest) < 0) {
close(fd);
f->backend_fd = -1;
return -1;
}
f->backend_epoll_registered = 1;
}
}
return 0;
}
SP_TEST_API static int sp_ns_backend_wants_read(const struct sp_ns_flow *f) {
uint64_t represented;
if (f->use_socks && !f->established)
return 1;
if (f->socks_residual_active || f->egress.eof_reported)
return 0;
represented = f->egress.stream_tail >= f->egress.stream_consumed
? f->egress.stream_tail - f->egress.stream_consumed
: 0;
return !(f->backend_read_idle && represented > 0);
}
SP_TEST_API static uint32_t
sp_ns_backend_observation_events(const struct sp_ns_flow *f) {
uint32_t events = EPOLLERR;
if (f == NULL || !f->egress.eof_reported)
events |= EPOLLRDHUP | EPOLLHUP;
return events;
}
SP_TEST_API static int
sp_ns_backend_should_unregister(const struct sp_ns_flow *f) {
return f != NULL && f->egress.eof_reported;
}
/* Updates backend epoll interest: readable while new stream input is possible,
* writable only while an ingress reservation is pending re-flush. Returns -1
* when the kernel registration could not be updated; the caller must then
* reset the flow (F2). */
static int sp_ns_update_backend_interest(struct sp_ns_flow *f) {
uint32_t want = sp_ns_backend_observation_events(f);
struct epoll_event ev;
size_t index = (size_t)(f - g_ns_flows);
if (f->backend_fd < 0 || g_epfd < 0)
return 0;
/* EPOLLHUP and EPOLLERR are reported even when omitted from the requested
* mask. After EOF reaches the core, leave the fd open for authorized
* consume effects but remove it from epoll so terminal readiness cannot
* spin the level-triggered loop. */
if (sp_ns_backend_should_unregister(f)) {
if (f->backend_epoll_registered) {
epoll_del(f->backend_fd);
f->backend_epoll_registered = 0;
f->backend_interest = (struct epoll_event){0};
}
return 0;
}
if (!f->backend_epoll_registered)
return 0;
/* Watch backend readability unless a prior observe made no progress *and* we
* are still holding unconsumed flight (peeked bytes the child has not ACKed).
* In that state level-triggered EPOLLIN re-fires on the same peeked data with
* nothing new to read, spinning the loop at 100% CPU; a child ACK that
* consumes flight clears backend_read_idle and re-arms EPOLLIN through the
* pump. When represented is 0 (nothing in flight) EPOLLIN stays armed until
* EOF is reported; a closed backend must never be re-armed because its
* level-triggered EOF readiness would otherwise spin the broker forever. */
if (sp_ns_backend_wants_read(f))
want |= EPOLLIN;
if (f->connect_effect_id != 0 || f->ingress.pending_bytes > 0 ||
(f->use_socks && socks_has_pending_tx(&f->cold->socks)))
want |= EPOLLOUT;
ev.events = want;
ev.data.u64 =
epoll_registration_token(FD_TCP_CORE, index, f->backend_fd, f->generation);
if (sp_epoll_update(EPOLL_CTL_MOD, f->backend_fd, &ev,
&f->backend_interest) < 0) {
sp_epoll_update_failed("TCP backend");
return -1;
}
return 0;
}
/* The live-adapter effect pump. Runs the core on an event, executes each emitted
* effect through the egress/ingress executors, and feeds every executor-produced result
* back as a NEW event on a bounded work queue, iterating to fixpoint. Executor
* results are internal events: a core rejection of one is a lockstep bug and is
* fatal, whereas rejection of an external child segment is a normal drop.
* Returns 0 to keep the flow, -1 to release it. */
static struct sp_pump_item
sp_pump_item_admission(const struct sp_ev_flow_admitted *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_ADMISSION, .now = input->now.value,
.value.admission = *input};
}
static struct sp_pump_item
sp_pump_item_segment(const struct sp_ev_tcp_segment *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_SEGMENT, .now = input->context.now.value,
.value.segment = *input};
}
static struct sp_pump_item
sp_pump_item_connect_result(const struct sp_ev_connect_result *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_CONNECT_RESULT, .now = input->context.now.value,
.value.connect_result = *input};
}
static struct sp_pump_item
sp_pump_item_stream_send_result(const struct sp_ev_stream_send_result *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_STREAM_SEND_RESULT, .now = input->context.now.value,
.value.stream_send_result = *input};
}
static struct sp_pump_item
sp_pump_item_stream_consumed(const struct sp_ev_stream_consumed *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_STREAM_CONSUMED, .now = input->context.now.value,
.value.stream_consumed = *input};
}
static struct sp_pump_item
sp_pump_item_stream_eof(const struct sp_ev_stream_eof *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_STREAM_EOF, .now = input->context.now.value,
.value.stream_eof = *input};
}
static struct sp_pump_item
sp_pump_item_tun_emit_result(const struct sp_ev_tun_emit_result *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_TUN_EMIT_RESULT, .now = input->context.now.value,
.value.tun_emit_result = *input};
}
static struct sp_pump_item
sp_pump_item_credit(const struct sp_ev_credit *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_CREDIT, .now = input->context.now.value,
.value.credit = *input};
}
static struct sp_pump_item
sp_pump_item_timer_fired(const struct sp_ev_timer_fired *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_TIMER_FIRED, .now = input->context.now.value,
.value.timer_fired = *input};
}
static struct sp_pump_item
sp_pump_item_ingress_disposed(const struct sp_ev_ingress_disposed *input) {
return (struct sp_pump_item){.kind = SP_PUMP_ITEM_INGRESS_DISPOSED, .now = input->context.now.value,
.value.ingress_disposed = *input};
}
/* Exhaustiveness is a local security property for the only trusted runtime
* event discriminant. Make enum growth a build error here without imposing
* -Wswitch-enum on unrelated legacy protocol switches. */
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic error "-Wswitch"
#pragma GCC diagnostic error "-Wswitch-enum"
#endif
static enum sp_tcp_core_result sp_pump_item_dispatch(
struct sp_tcp_core *core, const struct sp_pump_item *item,
struct sp_typed_effect_list *effects) {
switch (item->kind) {
case SP_PUMP_ITEM_ADMISSION:
return sp_tcp_core_admit(core, &item->value.admission, effects);
case SP_PUMP_ITEM_SEGMENT:
return sp_tcp_core_segment(core, &item->value.segment, effects);
case SP_PUMP_ITEM_CONNECT_RESULT:
return sp_tcp_core_connect_result(core, &item->value.connect_result,
effects);
case SP_PUMP_ITEM_STREAM_SEND_RESULT:
return sp_tcp_core_stream_send_result(
core, &item->value.stream_send_result, effects);
case SP_PUMP_ITEM_STREAM_CONSUMED:
return sp_tcp_core_stream_consumed(core, &item->value.stream_consumed,
effects);
case SP_PUMP_ITEM_STREAM_EOF:
return sp_tcp_core_stream_eof(core, &item->value.stream_eof, effects);
case SP_PUMP_ITEM_TUN_EMIT_RESULT:
return sp_tcp_core_tun_emit_result(core, &item->value.tun_emit_result,
effects);
case SP_PUMP_ITEM_CREDIT:
return sp_tcp_core_credit(core, &item->value.credit, effects);
case SP_PUMP_ITEM_TIMER_FIRED:
return sp_tcp_core_timer_fired(core, &item->value.timer_fired, effects);
case SP_PUMP_ITEM_INGRESS_DISPOSED:
return sp_tcp_core_ingress_disposed(core, &item->value.ingress_disposed,
effects);
}
return SP_TCP_CORE_BAD_EVENT;
}
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
static enum sp_tcp_core_result sp_ns_core_process(
struct sp_ns_flow *f, const struct sp_pump_item *item,
struct sp_typed_effect_list *effects) {
enum sp_tcp_core_result result;
f->challenge_ack = (uint8_t)(item->kind == SP_PUMP_ITEM_SEGMENT &&
sp_ns_segment_needs_challenge_ack(&f->local_core, &item->value.segment));
result = sp_pump_item_dispatch(&f->local_core, item, effects);
sp_ns_timer_refresh(f);
return result;
}
static void sp_ns_local_core_init(struct sp_ns_flow *f) {
sp_tcp_core_init(&f->local_core);
}
/* Publish and child-originated TCP flows share the same local state-machine
* and executor ownership. Admission role, tuple, and SOCKS negotiation stay
* at their call sites; this helper only establishes the common adapters after
* flow_id/core_gen have been assigned. */
static void sp_ns_tcp_adapters_init(struct sp_ns_flow *f) {
sp_ns_local_core_init(f);
memset(&f->egress, 0, sizeof(f->egress));
f->egress.flow = f->flow_id;
f->egress.generation = f->core_gen;
f->egress.tcp_backend = 1;
sp_ingress_flow_init(&f->ingress, f->flow_id, f->core_gen);
}
static int sp_ns_pump_run(struct sp_ns_flow *f, struct sp_ns_pump *ctx,
uint64_t initial_staged_id);
static int sp_ns_publish_socks_residual(struct sp_ns_flow *f,
uint64_t now_ns);
enum {
SP_NS_PUMP_FAILED = -1,
SP_NS_PUMP_COMPLETE = 0,
};
static int sp_ns_pump_start(struct sp_ns_flow *f, struct sp_pump_item initial,
uint64_t staged_id, int insert_on_complete) {
struct sp_ns_pump *ctx = &g_ns_pump;
int tunfd = g_ns_tunfd_for_emit;
int rc;
if (ctx->owner != NULL) {
sp_counter_inc(&g_resource_counters.pump_reentry_failures);
return SP_NS_PUMP_FAILED;
}
ctx->owner = f;
ctx->tunfd = tunfd;
ctx->queue[0] = initial;
ctx->count = 1;
f->pump_close_graceful = 0;
rc = sp_ns_pump_run(f, ctx, staged_id);
ctx->owner = NULL;
ctx->count = 0;
ctx->tunfd = -1;
if (rc == SP_NS_PUMP_COMPLETE && insert_on_complete &&
sp_ns_tcp_table_insert(f) < 0)
return SP_NS_PUMP_FAILED;
if (rc == SP_NS_PUMP_COMPLETE && !g_ns_sched_servicing && tunfd >= 0)
sp_ns_service_tun_scheduler(tunfd);
if (rc == SP_NS_PUMP_COMPLETE &&
sp_ns_publish_socks_residual(f, sp_ns_now_ns()) < 0)
return SP_NS_PUMP_FAILED;
return rc;
}
static int sp_ns_pump_run(struct sp_ns_flow *f, struct sp_ns_pump *ctx,
uint64_t initial_staged_id) {
size_t qhead = 0;
/* Capture the TUN execution descriptor for this bounded drain. */
g_ns_tunfd_for_emit = ctx->tunfd;
int iters = 0;
int release = 0;
while (ctx->count > 0 &&
iters < SP_NS_PUMP_ITERS) {
struct sp_pump_item item;
int external;
struct sp_typed_effect_list effects;
struct sp_pump_item produced_items[SP_TCP_CORE_MAX_EFFECTS];
size_t produced_count = 0;
enum sp_tcp_core_result core_rc = SP_TCP_CORE_OK;
uint64_t staged_id;
int authorized_staged = 0;
size_t i;
item = ctx->queue[qhead];
memset(&ctx->queue[qhead], 0, sizeof(ctx->queue[qhead]));
staged_id = initial_staged_id;
initial_staged_id = 0;
external = item.kind == SP_PUMP_ITEM_ADMISSION ||
item.kind == SP_PUMP_ITEM_SEGMENT;
qhead = (qhead + 1) % SP_NS_QUEUE_MAX;
ctx->count--;
/* Bound the immediate completion chain for this input. */
iters++;
core_rc = sp_ns_core_process(f, &item, &effects);
if (core_rc != SP_TCP_CORE_OK) {
if (external) {
if (staged_id != 0) {
struct sp_ingress_stage_slot *slot = sp_ingress_find_slot(
&g_ns_stage_pool, f->flow_id, f->core_gen, staged_id);
if (slot != NULL)
sp_ingress_stage_free(&f->ingress, slot);
}
continue; /* a rejected child segment is a normal drop */
}
/* The broker built this item itself; the core finding it malformed
* means broker and core state disagree (a lockstep bug). */
assert(core_rc != SP_TCP_CORE_BAD_EVENT);
DBG("internal item %d rejected", (int)item.kind);
return SP_NS_PUMP_FAILED;
}
for (i = 0; i < effects.count; ++i) {
const struct sp_typed_effect *effect = &effects.items[i];
struct sp_ev_stream_send_result send_result;
struct sp_ev_stream_consumed consumed_result;
struct sp_ev_tun_emit_result emit_result;
struct sp_ev_ingress_disposed disposed_result;
int produced = 0;
int emit_blocked = 0;
int rc;
switch (effect->type) {
case SP_FX_OPEN_STREAM:
if (sp_ns_open_backend(f, &effect->value.open_stream) < 0) {
DBG("backend open failed: %s", strerror(errno));
return SP_NS_PUMP_FAILED;
}
break;
case SP_FX_TCP_EMIT_CONTROL:
/* Challenge-ACK classification uses pre-event receive state. */
if (sp_ns_emit_child_record(f, &effect->value.tcp_emit_control,
f->challenge_ack) < 0)
return SP_NS_PUMP_FAILED;
break;
case SP_FX_TCP_EMIT_BATCH: {
const struct sp_fx_tcp_emit_batch *emit =
&effect->value.tcp_emit_batch;
struct sp_ns_emit_transaction emit_txn = {
.flow = f, .effect = emit, .tunfd = ctx->tunfd};
/* The preceding CREDIT confirmed these bytes are in the backend socket,
* so a materialize (peek) BLOCKED here is not expected; treat it as
* loud-fatal rather than a silent stall. */
/* The live trace shows the first batch materialized after socket
* consumption can be lost while its later successor reaches the child:
* the child remains at that batch's ACK boundary and dup-ACKs every
* later range. Keep SO_PEEK_OFF only while the socket head is still
* unchanged. Once any prefix was consumed (and for retransmission),
* use the bounded discard-iovec path, whose relative offset is explicit
* and cannot inherit kernel-specific peek-cursor state. */
if (f->socks_residual_active &&
emit->end.value <= f->cold->socks_residual_len) {
uint64_t residual_length = emit->end.value - emit->start.value;
if (emit->start.value > SIZE_MAX || residual_length > SIZE_MAX ||
emit->end.value > sizeof(f->cold->socks_residual)) {
errno = EOVERFLOW;
return SP_NS_PUMP_FAILED;
}
memcpy(g_ns_payload,
f->cold->socks_residual + (size_t)emit->start.value,
(size_t)residual_length);
rc = 1;
} else {
rc = sp_egress_materialize_emit(
&f->egress, f->backend_fd, emit,
f->egress.stream_consumed != 0 ||
f->local_core.tun_emit_effect.retransmission != 0,
g_ns_discard, sizeof(g_ns_discard), g_ns_payload,
sizeof(g_ns_payload));
}
if (rc <= 0) {
DBG("materialize_emit rc=%d (%s)", rc, strerror(errno));
return SP_NS_PUMP_FAILED;
}
rc = sp_egress_execute_emit_with_typed(
&f->egress, sp_ns_tun_write, &emit_txn, emit, ntohl(f->srv_ip),
ntohl(f->cli_ip), f->srv_port, f->cli_port, g_ns_payload,
(size_t)(emit->end.value - emit->start.value),
f->egress_force_software,
item.now, &emit_result);
if (rc < 0) {
DBG("emit_with fatal (%s)", strerror(errno));
return SP_NS_PUMP_FAILED;
}
if (rc == 0) {
/* The TUN output FIFO is full. Retain this exact effect and
* defer; the core holds its matching tun_emit_effect and will neither
* re-emit nor retransmit until the result arrives, so we re-execute
* *this* effect on TUN writability (sp_ns_retry_pending_emit). No
* result is produced, so snd-state does not advance and no lockstep
* control_seq is consumed. Stop processing further effects this pass;
* whatever the core queued behind the emit is re-derived on re-drive. */
f->pending_emit = *emit;
f->has_pending_emit = 1;
sp_ns_output_work_refresh(f);
emit_blocked = 1;
break;
}
if (f->has_sched_emit_result) {
DBG("multiple scheduler completions for flow %llu",
(unsigned long long)f->flow_id);
return SP_NS_PUMP_FAILED;
}
f->sched_emit_result = emit_result;
f->has_sched_emit_result = 1;
sp_ns_output_work_refresh(f);
break;
}
case SP_FX_STREAM_SEND: {
const struct sp_fx_stream_send *send = &effect->value.stream_send;
{
uint8_t *reservation =
sp_ns_reservation_acquire(&f->ingress, f->admission_role);
if (reservation == NULL) {
errno = ENOBUFS;
return SP_NS_PUMP_FAILED;
}
if (send->capability.value != 0) {
if (send->capability.value == staged_id)
authorized_staged = 1;
}
rc = sp_ingress_execute_stream_send_typed(
&g_ns_stage_pool, &f->ingress, sp_ingress_kernel_send,
(void *)(intptr_t)f->backend_fd, send, item.now,
reservation, SP_INGRESS_RESERVATION_SLOT_BYTES, &send_result);
if (rc < 0) {
DBG("stream_send fatal (%s)", strerror(errno));
return SP_NS_PUMP_FAILED;
}
if (rc == 0) {
/* STAGE_BLOCKED: bytes are reserved; re-flush on backend writability. */
break;
}
if (f->ingress.pending_bytes == 0)
sp_ns_reservation_release(&f->ingress);
produced_items[produced_count] =
sp_pump_item_stream_send_result(&send_result);
produced = 1;
break;
}
}
case SP_FX_STREAM_CONSUME: {
const struct sp_fx_stream_consume *consume =
&effect->value.stream_consume;
if (f->socks_residual_active &&
consume->start.value == f->egress.stream_consumed &&
consume->end.value <= f->cold->socks_residual_len) {
uint64_t accepted =
consume->end.value - consume->start.value;
if (accepted == 0 ||
UINT64_MAX - f->egress.stream_consumed < accepted)
return SP_NS_PUMP_FAILED;
f->egress.stream_consumed += accepted;
memset(&consumed_result, 0, sizeof(consumed_result));
sp_tcp_event_context_init(&consumed_result.context, f->flow_id,
f->core_gen, item.now);
consumed_result.effect = consume->context.effect;
consumed_result.accepted.value = accepted;
if (f->egress.stream_consumed == f->cold->socks_residual_len)
f->socks_residual_active = 0;
rc = 1;
} else {
rc = sp_egress_execute_consume_typed(
&f->egress, f->backend_fd, consume, item.now, 0,
g_ns_discard, sizeof(g_ns_discard), &consumed_result);
}
if (rc < 0) {
DBG("consume fatal (%s)", strerror(errno));
return SP_NS_PUMP_FAILED;
}
if (rc == 0)
break; /* BLOCKED: the authorized bytes will be drained on retry */
/* A child ACK freed flight: backend reads can make progress again, so
* re-arm EPOLLIN (cleared here; the post-pump interest recompute applies
* it). */
f->backend_read_idle = 0;
produced_items[produced_count] =
sp_pump_item_stream_consumed(&consumed_result);
produced = 1;
break;
}
case SP_FX_INGRESS_DISPOSE:
rc = sp_ingress_execute_dispose_typed(
&g_ns_stage_pool, &f->ingress, &effect->value.ingress_dispose,
item.now, &disposed_result);
if (rc < 0)
return SP_NS_PUMP_FAILED;
produced_items[produced_count] =
sp_pump_item_ingress_disposed(&disposed_result);
produced = 1;
break;
case SP_FX_STREAM_SHUTDOWN:
/* Backend-facing FIN; the child-facing close handshake is increment 3. */
(void)shutdown(f->backend_fd, SHUT_WR);
break;
case SP_FX_CLOSE_FLOW:
/* A close that follows a mutual FIN exchange is graceful; anything else
* (handshake/RTO/resource deadline, RST, connect or consume failure) is
* an abort that resets the child. */
if (f->local_core.peer_fin_seen && f->local_core.fin_acked)
f->pump_close_graceful = 1;
release = 1;
break;
}
if (produced) {
if (ctx->count + produced_count == SP_NS_QUEUE_MAX) {
DBG("pump work queue overflow");
return SP_NS_PUMP_FAILED;
}
produced_count++;
}
if (emit_blocked)
break; /* emit retained; skip effects queued behind it (re-derived) */
}
/* Prepend immediate completions in effect order before older work. */
while (produced_count > 0) {
qhead = (qhead + SP_NS_QUEUE_MAX - 1) % SP_NS_QUEUE_MAX;
ctx->queue[qhead] = produced_items[--produced_count];
ctx->count++;
}
/* Release child payloads not authorized by this event. */
if (staged_id != 0 && !authorized_staged) {
struct sp_ingress_stage_slot *slot = sp_ingress_find_slot(
&g_ns_stage_pool, f->flow_id, f->core_gen, staged_id);
if (slot != NULL) {
uint64_t interval_start = slot->interval_start;
uint64_t interval_end = slot->interval_end;
sp_ingress_stage_free(&f->ingress, slot);
g_ns_orphans_freed++;
DBG("freed unauthorized staged frame (flow %llu, "
"range=%llu..%llu event_seq=%u acked=%llu authorized=%llu, "
"orphans=%llu)",
(unsigned long long)f->flow_id,
(unsigned long long)interval_start,
(unsigned long long)interval_end,
item.kind == SP_PUMP_ITEM_SEGMENT ? item.value.segment.seq : 0,
(unsigned long long)f->local_core.rcv_acked_abs,
(unsigned long long)f->local_core.rx_authorized_end_abs,
(unsigned long long)g_ns_orphans_freed);
}
}
}
if (ctx->count != 0) {
DBG("pump iteration bound hit (runaway effect chain)");
return SP_NS_PUMP_FAILED;
}
if (release)
return SP_NS_PUMP_FAILED;
if (sp_ns_update_backend_interest(f) < 0)
return SP_NS_PUMP_FAILED;
return SP_NS_PUMP_COMPLETE;
}
/* The passive child handshake may still be in SYN-RECEIVED when the SOCKS
* proxy returns CONNECT success and coalesced application bytes. Keep those
* bytes private until the child's final ACK establishes the core, then publish
* their logical range exactly once. */
static int sp_ns_publish_socks_residual(struct sp_ns_flow *f,
uint64_t now_ns) {
struct sp_ev_credit event;
if (!f->socks_residual_active ||
f->local_core.state != SP_TCP_CORE_ESTABLISHED)
return 0;
if (f->egress.stream_tail == f->cold->socks_residual_len &&
f->local_core.stream_tail == f->cold->socks_residual_len)
return 0; /* already published; an ACK will retire the retained bytes */
if (f->egress.stream_tail != 0 || f->egress.stream_consumed != 0 ||
f->cold->socks_residual_len == 0)
return -1;
f->egress.stream_tail = f->cold->socks_residual_len;
memset(&event, 0, sizeof(event));
event.context.key = sp_flow_key_make(
(sp_flow_id){.value = f->flow_id},
(sp_generation_id){.value = f->core_gen});
event.context.now.value = now_ns;
event.stream_tail.value = f->cold->socks_residual_len;
event.tun_segments.value = SP_TCP_CORE_MAX_SEGMENTS;
return sp_ns_pump_start(f, sp_pump_item_credit(&event), 0, 0);
}
static int sp_publish_synthetic_port_in_use(uint16_t port) {
for (size_t i = 0; i < sp_ns_tcp_slot_limit(); ++i)
if (g_ns_flows[i].in_use && g_ns_flows[i].srv_ip == PUBLISH_SYNTHETIC_IP &&
g_ns_flows[i].srv_port == port)
return 1;
for (size_t i = 0; i < MAX_PUBLISH_UDP; ++i)
if (publish_udp_flows[i].used && publish_udp_flows[i].synthetic_port == port)
return 1;
for (size_t i = 0; i < SP_TIMEWAIT_ALL_MAX; ++i)
if (g_timewait_map.entries[i].state == SP_NS_TIMEWAIT_LIVE &&
g_timewait_map.entries[i].srv_ip == PUBLISH_SYNTHETIC_IP &&
g_timewait_map.entries[i].srv_port == port)
return 1;
return 0;
}
static size_t sp_ns_publish_live_limit(void) {
return sp_publish_limit_for_shared(sp_ns_tcp_slot_limit());
}
static int sp_ns_publish_connect_capacity(void) {
size_t live = 0;
size_t half_open = 0;
for (size_t i = 0; i < sp_ns_tcp_slot_limit(); ++i) {
const struct sp_ns_flow *flow = &g_ns_flows[i];
if (!flow->in_use || flow->admission_role != SP_TCP_ROLE_PUBLISH)
continue;
live++;
if (flow->local_core.state < SP_TCP_CORE_ESTABLISHED)
half_open++;
}
return live < sp_ns_publish_live_limit() &&
half_open < SP_TCP_HALF_PUBLISH_MAX &&
half_open < SP_CONNECT_PUBLISH_MAX;
}
static int sp_ns_accept_publish(int tunfd, struct publish_rule *rule, int fd,
const struct sockaddr_in *peer) {
struct sp_ns_flow *f;
struct sp_ev_flow_admitted event;
struct sp_admission_key source_key;
uint16_t synthetic_port;
uint32_t iss;
if (rule == NULL || peer == NULL || fd < 0)
return -1;
if (!sp_ns_publish_connect_capacity()) {
sp_counter_inc(&g_resource_counters.tcp_flow_rejections);
return -1;
}
synthetic_port = sp_ns_publish_port_alloc();
if (synthetic_port == 0 ||
sp_random_exact_nonblocking(&iss, sizeof(iss)) < 0) {
sp_counter_inc(&g_resource_counters.tcp_flow_rejections);
return -1;
}
f = sp_ns_alloc();
if (f == NULL) {
sp_counter_inc(&g_resource_counters.tcp_flow_rejections);
return -1;
}
memset(f, 0, sizeof(*f));
f->cold = &g_ns_cold_flows[f - g_ns_flows];
memset(f->cold, 0, sizeof(*f->cold));
f->in_use = 1;
f->backend_fd = fd;
f->cli_ip = SANDBOX_IP;
f->srv_ip = PUBLISH_SYNTHETIC_IP;
f->cli_port = rule->container_port;
f->srv_port = synthetic_port;
f->admission_role = SP_TCP_ROLE_PUBLISH;
f->admission_source_ip = peer->sin_addr.s_addr;
f->flow_id = g_ns_next_flow_id++;
f->core_gen = 1;
f->established = 1; /* accepted host socket needs no connect completion */
f->last_active_ns = sp_ns_now_ns();
sp_ns_timer_refresh(f);
f->generation = g_ns_epoll_generation;
g_ns_epoll_generation = epoll_next_generation(g_ns_epoll_generation);
sp_ns_tcp_adapters_init(f);
sp_resource_admission_source_key(&source_key, f->admission_source_ip,
f->admission_role);
{
struct sp_admission_entry *source =
sp_resource_admission_acquire(&source_key);
if (source == NULL ||
!sp_resource_admission_take(source, sp_ns_now_ns())) {
if (source != NULL)
sp_resource_admission_release(source);
sp_ns_flow_release(f);
return 0;
}
}
f->admission_source_held = 1;
{
size_t index = (size_t)(f - g_ns_flows);
struct epoll_event ev = {
.events = EPOLLRDHUP | EPOLLERR | EPOLLHUP,
.data.u64 = epoll_registration_token(FD_TCP_CORE, index, fd,
f->generation)};
if (g_epfd >= 0) {
if (sp_epoll_update(EPOLL_CTL_ADD, fd, &ev, &f->backend_interest) < 0) {
sp_ns_flow_release(f);
return 0;
}
f->backend_epoll_registered = 1;
}
}
memset(&event, 0, sizeof(event));
event.key = sp_flow_key_make((sp_flow_id){.value = f->flow_id},
(sp_generation_id){.value = f->core_gen});
event.now.value = sp_ns_now_ns();
event.tuple.local_addr = ntohl(f->srv_ip);
event.tuple.peer_addr = ntohl(f->cli_ip);
event.tuple.local_port = f->srv_port;
event.tuple.peer_port = f->cli_port;
event.policy_role = SP_TCP_ROLE_PUBLISH;
event.open_role = SP_TCP_OPEN_ACTIVE;
event.local_iss = iss;
event.local_rcv_wscale = SP_TCP_RCV_WSCALE;
g_ns_tunfd_for_emit = tunfd;
if (sp_ns_pump_start(f, sp_pump_item_admission(&event), 0, 1) < 0) {
sp_ns_flow_release(f);
g_ns_tunfd_for_emit = -1;
return 0;
}
if (sp_ns_update_backend_interest(f) < 0)
sp_ns_reset_child(f);
g_ns_tunfd_for_emit = -1;
return 0;
}
/* Resolve a carried scheduler owner index without reversing a flow id through
* the live table. The index is merely a hint until all three identity fields
* agree; this is the generation-reuse/type-confusion guard at the final sink. */
static struct sp_ns_flow *
sp_ns_sched_flow_for_record(const struct sp_scheduler_record *record) {
uint16_t index;
if (record == NULL || record->flow_index == UINT16_MAX)
return NULL;
index = record->flow_index;
if ((size_t)index >= sp_ns_tcp_slot_limit()) {
sp_counter_inc(&g_resource_counters.scheduler_stale_index_rejects);
return NULL;
}
if (!g_ns_flows[index].in_use ||
g_ns_flows[index].flow_id != record->flow ||
g_ns_flows[index].core_gen != record->generation) {
sp_counter_inc(&g_resource_counters.scheduler_stale_index_rejects);
return NULL;
}
return &g_ns_flows[index];
}
/* Pure: decides from the record, its live owner `f` (NULL when there is
* none) and FIFO fullness alone, before anything is written. Returns 1 when
* the frame should be written; otherwise *verdict is the outcome. A record
* whose live owner was reset or reused is dropped as written; CLOSED/ICMP-error
* responses are auxiliary control records with no live flow by design. */
static int sp_ns_sched_decide(const struct sp_scheduler_record *record,
const struct sp_ns_flow *f, int fifo_full,
int *verdict) {
int aux = record->klass == SP_SCHEDULER_CLASS_CONTROL &&
sp_ns_is_aux_flow(record->flow);
*verdict = SP_SCHEDULER_WRITE_BLOCKED;
if (f == NULL && !aux) {
*verdict = SP_SCHEDULER_WRITE_DONE;
return 0;
}
if (fifo_full) {
/* A GSO container is retired and re-executed as scalars instead. */
if (record->klass == SP_SCHEDULER_CLASS_DATA && f->sched_emit_outstanding == 1)
*verdict = SP_SCHEDULER_WRITE_RETIRED;
return 0;
}
if (f == NULL)
return 1;
/* Every queued data record is counted outstanding, and a terminal record's
* core result is held until it is written. */
if (record->klass == SP_SCHEDULER_CLASS_DATA)
assert(f->sched_emit_outstanding > 0);
else if (f->sched_terminal_seq == record->seq_abs)
assert(f->has_sched_emit_result);
return 1;
}
static int sp_ns_sched_write(void *context,
const struct sp_scheduler_record *record,
const uint8_t *frame) {
int tunfd = *(int *)context;
struct sp_ns_flow *f = sp_ns_sched_flow_for_record(record);
int verdict;
if (!sp_ns_sched_decide(record, f, g_tun_queue.count == TUN_QUEUE_PACKETS,
&verdict)) {
if (verdict == SP_SCHEDULER_WRITE_RETIRED) {
/* Retire without completing the core effect; the exact effect is
* re-executed through the ordered scalar path after the final FIFO
* drains, producing independently scheduled scalar records. */
f->sched_emit_outstanding = 0;
f->sched_restart_software = 1;
f->egress_force_software = 1;
f->has_pending_emit = 1;
sp_ns_output_work_refresh(f);
f->sched_emit_written = 0;
}
return verdict;
}
if (tun_write_frame(tunfd, frame, record->frame_len,
record->klass == SP_SCHEDULER_CLASS_CONTROL ? "scheduler control"
: "scheduler data") < 0)
return errno == EAGAIN || errno == EWOULDBLOCK ? SP_SCHEDULER_WRITE_BLOCKED
: SP_SCHEDULER_WRITE_FAILED;
if (record->klass == SP_SCHEDULER_CLASS_DATA) {
sp_scheduler_commit_seq(&g_ns_tun_sched, record->flow, record->seq_abs);
f->sched_emit_outstanding--;
if (f->sched_emit_outstanding == 0)
f->sched_emit_written = 1;
} else if (f != NULL && f->sched_terminal_seq == record->seq_abs) {
f->sched_terminal_seq = 0;
f->sched_emit_written = 1;
}
return SP_SCHEDULER_WRITE_DONE;
}
static void sp_ns_service_tun_scheduler(int tunfd) {
size_t i;
if (g_ns_sched_servicing || g_broker_io_failed)
return;
g_ns_sched_servicing = 1;
if (sp_scheduler_service(&g_ns_tun_sched, sp_ns_sched_write, &tunfd,
sp_ns_scheduler_clock, NULL) == SIZE_MAX) {
g_broker_io_failed = 1;
g_ns_sched_servicing = 0;
return;
}
/* A final-sink-blocked GSO container was retired without a core completion.
* Re-execute its exact effect through the scalar fallback, then service the
* independently admitted scalar records. */
for (i = sp_ns_output_work_next(0); i < sp_ns_tcp_slot_limit();
i = sp_ns_output_work_next(i + 1)) {
struct sp_ns_flow *f = &g_ns_flows[i];
if (!f->in_use || !f->sched_restart_software)
continue;
f->sched_restart_software = 0;
f->sched_emit_written = 0;
sp_ns_retry_pending_emit(tunfd, f);
}
/* Credit epochs: an admission-blocked producer is not re-driven merely
* because the TUN fd remained writable. Retry only after scheduler service
* published a greater epoch with growth in a dimension that actually
* blocked this record. The scan happens after service returns, so admission
* cannot re-enter a mutating scheduler callback. */
for (i = sp_ns_output_work_next(0); i < sp_ns_tcp_slot_limit();
i = sp_ns_output_work_next(i + 1)) {
struct sp_ns_flow *f = &g_ns_flows[i];
struct sp_scheduler_credit credit;
if (!f->in_use || !f->has_pending_emit || !f->waits_sched_credit)
continue;
credit = sp_scheduler_credit_now(&g_ns_tun_sched);
if (!sp_scheduler_credit_permits_retry(&f->sched_blocked_credit, &credit,
f->sched_blocked_dims))
continue;
f->waits_sched_credit = 0;
f->sched_blocked_dims = 0;
sp_ns_retry_pending_emit(tunfd, f);
}
/* Only now is the executor result visible to the core: the scheduler write sink
* accepted the exact record, so send-state can safely advance. */
for (i = sp_ns_output_work_next(0); i < sp_ns_tcp_slot_limit();
i = sp_ns_output_work_next(i + 1)) {
struct sp_ns_flow *f = &g_ns_flows[i];
int saved_tunfd;
if (!f->in_use || !f->has_sched_emit_result || !f->sched_emit_written)
continue;
f->has_sched_emit_result = 0;
f->sched_emit_written = 0;
/* Timestamp final-sink acceptance, not the original queued emission. */
f->sched_emit_result.context.now.value = sp_ns_now_ns();
saved_tunfd = g_ns_tunfd_for_emit;
g_ns_tunfd_for_emit = tunfd;
if (sp_ns_pump_start(f, sp_pump_item_tun_emit_result(&f->sched_emit_result),
0, 0) < 0)
sp_ns_teardown(f);
else if (f->in_use)
sp_ns_maybe_park_timewait(f);
g_ns_tunfd_for_emit = saved_tunfd;
sp_ns_output_work_refresh(f);
}
g_ns_sched_servicing = 0;
g_ns_sched_tun_pending =
g_ns_tun_sched.control.count != 0 || g_ns_tun_sched.data.count != 0;
tun_update_events(tunfd);
}
/* CLOSED dispatch remains outside the TCP core: there
* is no TCB to mutate. It constructs the RFC CLOSED response canonically, then
* the overload policy applies the closed-RST bucket and the never-RST-to-RST rule before
* the scheduler owns final TUN delivery. */
static int sp_ns_emit_closed_rst(int tunfd, uint32_t cli_ip, uint32_t srv_ip,
uint16_t cli_port, uint16_t srv_port,
uint32_t seq, uint32_t ack, uint8_t flags,
size_t payload_len, uint64_t now_ns) {
uint8_t record[SP_VNET_HDR_BYTES + 40];
uint8_t response_flags;
uint32_t response_seq;
uint32_t response_ack;
uint64_t flow = sp_ns_aux_flow_id(ntohl(cli_ip), ntohl(srv_ip), cli_port,
srv_port, IPPROTO_TCP);
size_t record_len;
if ((flags & SP_TCP_CORE_RST) != 0)
return sp_ns_admit_aux_response(SP_SCHEDULER_RESP_CLOSED_RST, flow, 1, NULL, 0,
now_ns);
if ((flags & SP_TCP_CORE_ACK) != 0) {
response_flags = SP_TCP_CORE_RST;
response_seq = ack;
response_ack = 0;
} else {
size_t sequence_len = payload_len;
if ((flags & SP_TCP_CORE_SYN) != 0)
sequence_len++;
if ((flags & SP_TCP_CORE_FIN) != 0)
sequence_len++;
response_flags = SP_TCP_CORE_RST | SP_TCP_CORE_ACK;
response_seq = 0;
response_ack = tcp_seq_add_len(seq, sequence_len);
}
record_len = sp_vnet_build_tcp_record(
record, sizeof(record), ntohl(srv_ip), ntohl(cli_ip), srv_port,
cli_port, response_seq, response_ack, response_flags, NULL, 0, 0);
if (record_len == 0)
return -1;
if (sp_ns_admit_aux_response(SP_SCHEDULER_RESP_CLOSED_RST, flow, 0, record,
record_len, now_ns) < 0)
return -1;
sp_ns_service_tun_scheduler(tunfd);
return 0;
}
static int sp_udp_emit_error(int tunfd, const uint8_t *quote,
const struct udp_flow *f,
enum sp_udp_send_result result) {
uint8_t record[SP_VNET_HDR_BYTES + 56];
uint16_t rejected_len;
uint16_t mtu = 0;
uint16_t quoted_sport;
uint16_t quoted_dport;
uint64_t flow;
int saved_tunfd;
struct sp_icmp_control_request request;
struct sp_scalar_packet reply;
if (quote == NULL || f == NULL ||
(result != SP_UDP_SEND_EMSGSIZE && result != SP_UDP_SEND_REFUSED) ||
quote[0] != 0x45 || quote[9] != IPPROTO_UDP ||
(sp_get_be16(quote + 6) & (IP_MF | IP_OFFMASK)) != 0)
return -1;
rejected_len = sp_get_be16(quote + 2);
quoted_sport = sp_get_be16(quote + SP_IPV4_HEADER_BYTES);
quoted_dport = sp_get_be16(quote + SP_IPV4_HEADER_BYTES + 2);
if (sp_get_be32(quote + 12) != ntohl(f->cli_ip) ||
sp_get_be32(quote + 16) != ntohl(f->srv_ip) ||
quoted_sport != f->cli_port || quoted_dport != f->srv_port)
return -1;
if (result == SP_UDP_SEND_EMSGSIZE) {
mtu = udp_effective_path_mtu(f, rejected_len);
if (mtu == 0) {
sp_counter_inc(&g_resource_counters.icmp_drops);
return 0;
}
}
memset(&request, 0, sizeof(request));
request.code = result == SP_UDP_SEND_EMSGSIZE ? 4 : 3;
request.mtu = mtu;
memcpy(request.quote, quote, SP_IPV4_HEADER_BYTES + SP_UDP_HEADER_BYTES);
if (sp_build_icmp_control_reply(&request, sp_icmp_reply_id++, &reply) !=
SP_CODEC_OK) {
sp_counter_inc(&g_resource_counters.icmp_drops);
return -1;
}
memset(record, 0, sizeof(record));
memcpy(record + SP_VNET_HDR_BYTES, reply.bytes, reply.len);
flow = sp_ns_aux_flow_id(ntohl(f->cli_ip), ntohl(f->srv_ip), f->cli_port,
f->srv_port, IPPROTO_UDP);
saved_tunfd = g_ns_tunfd_for_emit;
g_ns_tunfd_for_emit = tunfd;
if (sp_ns_admit_aux_response(SP_SCHEDULER_RESP_ICMP, flow, 0, record,
SP_VNET_HDR_BYTES + reply.len,
sp_ns_now_ns()) < 0) {
g_ns_tunfd_for_emit = saved_tunfd;
sp_counter_inc(&g_resource_counters.icmp_drops);
return -1;
}
sp_ns_service_tun_scheduler(tunfd);
g_ns_tunfd_for_emit = saved_tunfd;
return 0;
}
static int sp_ns_timewait_segment_acceptable(
const struct sp_ns_timewait_entry *entry, uint64_t seq_abs,
uint64_t sequence_len) {
if (sequence_len == 0) {
if (entry->rcv_adv_right_abs == entry->rcv_acked_abs)
return seq_abs == entry->rcv_acked_abs;
return seq_abs >= entry->rcv_acked_abs &&
seq_abs <= entry->rcv_adv_right_abs;
}
return seq_abs < entry->rcv_adv_right_abs &&
seq_abs + sequence_len > entry->rcv_acked_abs;
}
/* A TIME-WAIT tombstone is not a reduced live flow. It cannot reopen a
* backend or accept stream bytes; it only preserves the quarantined tuple's
* required ACK/challenge/RST disposition until its absolute expiry. */
static int sp_ns_handle_timewait(int tunfd, struct sp_ns_timewait_entry *entry,
uint32_t seq, uint8_t flags,
size_t payload_len, uint64_t now_ns) {
uint64_t sequence_len = payload_len;
uint64_t seq_abs;
int acceptable;
uint8_t record[SP_VNET_HDR_BYTES + 40];
size_t record_len;
enum sp_scheduler_response_kind kind = SP_SCHEDULER_RESP_CURRENT_ACK;
if (entry == NULL || entry->state != SP_NS_TIMEWAIT_LIVE)
return -1;
if ((flags & SP_TCP_CORE_SYN) != 0)
sequence_len++;
if ((flags & SP_TCP_CORE_FIN) != 0)
sequence_len++;
seq_abs = sp_tcp_unwrap_u32(seq, entry->rcv_acked_abs);
acceptable = sp_ns_timewait_segment_acceptable(entry, seq_abs, sequence_len);
if ((flags & SP_TCP_CORE_RST) != 0) {
if (acceptable && seq_abs == entry->rcv_acked_abs)
sp_ns_timewait_remove(entry);
else if (acceptable)
kind = SP_SCHEDULER_RESP_CHALLENGE_ACK;
else
return 0; /* unacceptable RST is silent */
} else if ((flags & SP_TCP_CORE_SYN) != 0 && acceptable) {
kind = SP_SCHEDULER_RESP_CHALLENGE_ACK;
}
/* RFC TIME-WAIT restart applies only to an acceptable retransmitted FIN. */
if ((flags & SP_TCP_CORE_FIN) != 0 && acceptable)
sp_ns_timewait_heap_refresh(entry, now_ns + SP_TCP_TIME_WAIT_NS);
if ((flags & SP_TCP_CORE_RST) != 0 && kind == SP_SCHEDULER_RESP_CURRENT_ACK)
return 0; /* exact RST removed the tombstone and has no response */
record_len = sp_vnet_build_tcp_record(
record, sizeof(record), ntohl(entry->srv_ip), ntohl(entry->cli_ip),
entry->srv_port, entry->cli_port, (uint32_t)entry->snd_nxt_abs,
(uint32_t)entry->rcv_acked_abs, SP_TCP_CORE_ACK, NULL, 0, 0);
if (record_len == 0)
return -1;
if (sp_ns_admit_aux_response(
kind, sp_ns_timewait_hash(entry->cli_ip, entry->srv_ip,
entry->cli_port, entry->srv_port),
0, record, record_len, now_ns) < 0)
return -1;
sp_ns_service_tun_scheduler(tunfd);
return 0;
}
/* Coalesce contiguous scalar child packets into the same immutable ingress slot
* and one core event. This is the live counterpart of a TCPv4 GSO frame:
* bytes remain executor-owned, while the engine sees one contiguous sequence
* interval and can authorize it with one capability/effect round trip. */
static void
sp_ns_drop_unstaged_payload(struct sp_ev_tcp_segment *segment) {
segment->ingress.value = 0;
segment->payload_length.value = 0;
segment->flags &= (uint8_t)~SP_TCP_CORE_FIN;
}
/* Handles a validated child->broker TCP packet under the core adapter. The
* immutable parser product and its backing bytes are consumed synchronously;
* neither pointer is retained. Returns 1 if the packet was consumed by this
* path, 0 when the handoff contract is not satisfied. */
static int handle_core_tcp(int tunfd, const uint8_t *pkt, size_t len,
const struct sp_packet_value *parsed) {
const struct sp_tcp_value *tcp;
uint32_t cli_ip;
uint32_t srv_ip;
uint16_t cli_port;
uint16_t srv_port;
uint8_t flags;
size_t payload_off;
size_t payload_len;
struct sp_ns_flow *f;
uint64_t now = sp_ns_now_ns();
if (pkt == NULL || parsed == NULL ||
parsed->ip.protocol != IPPROTO_TCP ||
parsed->ip.src != ntohl(SANDBOX_IP) ||
(size_t)parsed->ip.total_len != len ||
parsed->ip.transport_offset > len ||
parsed->ip.transport_len != len - parsed->ip.transport_offset ||
parsed->tcp.data_offset < 5 || parsed->tcp.data_offset > 15 ||
parsed->tcp.payload_offset !=
parsed->ip.transport_offset +
(size_t)parsed->tcp.data_offset * 4U ||
!sp_slice_ok(len, parsed->tcp.payload_offset,
parsed->tcp.payload_len) ||
parsed->tcp.payload_len != len - parsed->tcp.payload_offset)
return 0;
tcp = &parsed->tcp;
cli_ip = htonl(parsed->ip.src);
srv_ip = htonl(parsed->ip.dst);
cli_port = tcp->src_port;
srv_port = tcp->dst_port;
flags = (uint8_t)(tcp->flags & 0x3fU);
payload_off = tcp->payload_offset;
payload_len = tcp->payload_len;
g_ns_tunfd_for_emit = tunfd;
f = sp_ns_find(cli_ip, cli_port, srv_ip, srv_port);
/* Published tuples are allocated only by an accepted host connection. A
* child packet cannot create or rebind one, regardless of global egress
* mode. */
if (srv_ip == PUBLISH_SYNTHETIC_IP && f == NULL) {
g_ns_tunfd_for_emit = -1;
return 1;
}
if (srv_ip != PUBLISH_SYNTHETIC_IP &&
g_net_policy.egress != EGRESS_DIRECT &&
g_net_policy.egress != EGRESS_SOCKS) {
g_ns_tunfd_for_emit = -1;
return 0;
}
/* TIME-WAIT is a separate bounded tombstone, not a reusable live-flow
* slot. It is consulted in the same dispatch turn as the live lookup so a
* tuple cannot be allocated between the two ownership decisions. */
if (f == NULL) {
struct sp_ns_timewait_entry *timewait =
sp_ns_timewait_find_live(cli_ip, srv_ip, cli_port, srv_port, now);
if (timewait != NULL) {
int rc = sp_ns_handle_timewait(
tunfd, timewait, tcp->seq, flags, payload_len, now);
g_ns_tunfd_for_emit = -1;
return rc < 0 ? -1 : 1;
}
}
if (f == NULL) {
struct sp_ev_flow_admitted admission;
uint8_t admission_role;
if ((flags & SP_TCP_FLAG_SYN) == 0 ||
(flags & (SP_TCP_FLAG_ACK | SP_TCP_FLAG_RST)) != 0) {
if (sp_ns_emit_closed_rst(tunfd, cli_ip, srv_ip, cli_port, srv_port,
tcp->seq, tcp->ack, flags, payload_len,
now) < 0)
{
g_ns_tunfd_for_emit = -1;
return -1;
}
g_ns_tunfd_for_emit = -1;
return 1;
}
if ((is_gateway_ip(srv_ip) &&
!sp_policy_gateway_allowed(&g_net_policy, srv_ip, srv_port,
IPPROTO_TCP)) ||
(!is_gateway_ip(srv_ip) && g_net_policy.egress == EGRESS_DIRECT &&
!sp_policy_direct_egress_allowed(&g_net_policy, srv_ip, srv_port,
IPPROTO_TCP))) {
log_direct_egress_block(srv_ip, srv_port, IPPROTO_TCP);
if (sp_ns_emit_closed_rst(tunfd, cli_ip, srv_ip, cli_port, srv_port,
tcp->seq, tcp->ack, flags, payload_len,
now) < 0)
{
g_ns_tunfd_for_emit = -1;
return -1;
}
g_ns_tunfd_for_emit = -1;
return 1;
}
admission_role = (uint8_t)(g_net_policy.egress == EGRESS_SOCKS &&
!is_gateway_ip(srv_ip)
? SP_TCP_ROLE_SOCKS
: SP_TCP_ROLE_DIRECT);
if (!sp_ns_tcp_admission_capacity(admission_role)) {
sp_counter_inc(&g_resource_counters.tcp_flow_rejections);
(void)sp_ns_emit_closed_rst(
tunfd, cli_ip, srv_ip, cli_port, srv_port, tcp->seq, tcp->ack,
flags, payload_len, now);
g_ns_tunfd_for_emit = -1;
return 1;
}
f = sp_ns_alloc();
if (f == NULL) {
if (sp_ns_emit_closed_rst(tunfd, cli_ip, srv_ip, cli_port, srv_port,
tcp->seq, tcp->ack, flags, payload_len,
now) < 0)
{
g_ns_tunfd_for_emit = -1;
return -1;
}
g_ns_tunfd_for_emit = -1;
return 1;
}
memset(f, 0, sizeof(*f));
f->cold = &g_ns_cold_flows[f - g_ns_flows];
memset(f->cold, 0, sizeof(*f->cold));
f->in_use = 1;
f->backend_fd = -1;
f->cli_ip = cli_ip;
f->srv_ip = srv_ip;
f->cli_port = cli_port;
f->srv_port = srv_port;
f->use_socks = g_net_policy.egress == EGRESS_SOCKS &&
!is_gateway_ip(srv_ip);
f->admission_role = admission_role;
f->admission_source_ip = cli_ip;
{
struct sp_admission_key source_key;
sp_resource_admission_source_key(&source_key, cli_ip, f->admission_role);
struct sp_admission_entry *source =
sp_resource_admission_acquire(&source_key);
if (source == NULL || !sp_resource_admission_take(source, now)) {
if (source != NULL)
sp_resource_admission_release(source);
sp_ns_flow_release(f);
(void)sp_ns_emit_closed_rst(
tunfd, cli_ip, srv_ip, cli_port, srv_port, tcp->seq, tcp->ack,
flags, payload_len, now);
g_ns_tunfd_for_emit = -1;
return 1;
}
f->admission_source_held = 1;
}
if (f->use_socks && g_net_policy.socks_remote_dns) {
struct sp_dns_ref mapping;
struct sp_dns_name_view name;
if (sp_dns_core_acquire_ip(&g_dns_core, srv_ip, now, &mapping) == 0) {
f->dns_mapping_ref = mapping;
if (sp_dns_core_mapping_name(&g_dns_core, mapping, &name) < 0 ||
name.name_len >= sizeof(f->cold->socks_target_domain)) {
sp_ns_flow_release(f);
g_ns_tunfd_for_emit = -1;
return 1;
}
memcpy(f->cold->socks_target_domain, name.name, name.name_len);
f->cold->socks_target_domain[name.name_len] = '\0';
} else if (dns_is_synthetic_ip(srv_ip)) {
sp_ns_flow_release(f);
g_ns_tunfd_for_emit = -1;
return 1;
}
}
f->flow_id = g_ns_next_flow_id++;
f->core_gen = 1;
f->last_active_ns = sp_ns_now_ns();
sp_ns_timer_refresh(f);
sp_ns_tcp_adapters_init(f);
memset(&admission, 0, sizeof(admission));
admission.key.flow.value = f->flow_id;
admission.key.generation.value = f->core_gen;
admission.now.value = now;
admission.tuple.local_addr = ntohl(srv_ip);
admission.tuple.peer_addr = ntohl(cli_ip);
admission.tuple.local_port = srv_port;
admission.tuple.peer_port = cli_port;
admission.policy_role = f->use_socks ? SP_TCP_ROLE_SOCKS
: SP_TCP_ROLE_DIRECT;
admission.open_role = SP_TCP_OPEN_PASSIVE;
{
/* TCP ISNs remain unpredictable; backend creation needs no IPC token. */
uint32_t iss = 0;
if (sp_random_exact_nonblocking(&iss, sizeof(iss)) < 0) {
sp_counter_inc(&g_resource_counters.tcp_flow_rejections);
send_tcp_rst(tunfd, srv_ip, cli_ip, srv_port, cli_port, 0,
tcp_seq_add_len(tcp->seq, 1));
sp_ns_flow_release(f);
g_ns_tunfd_for_emit = -1;
return 1;
}
admission.local_iss = iss;
}
admission.local_rcv_wscale = SP_TCP_RCV_WSCALE;
admission.needs_connect = 1;
admission.passive_syn_present = 1;
admission.peer_isn = tcp->seq;
admission.peer_window = tcp->window;
admission.peer_mss = tcp->mss;
admission.peer_mss_present = tcp->has_mss;
admission.peer_ws_present = tcp->has_window_scale;
admission.peer_ws = tcp->window_scale;
if (sp_ns_pump_start(f, sp_pump_item_admission(&admission), 0, 1) < 0) {
send_tcp_rst(tunfd, srv_ip, cli_ip, srv_port, cli_port, 0,
tcp_seq_add_len(tcp->seq, 1));
sp_ns_flow_release(f);
}
g_ns_tunfd_for_emit = -1;
return 1;
}
/* Existing flow: feed the segment to the core. Child payload is staged into
* the global ingress pool first so the core can authorize an exact slice against
* it; if the core does not authorize it (a duplicate or out-of-window
* retransmit produces a current-ACK, not a STREAM_SEND), the orphaned slot is
* freed immediately after the pump so a burst of retransmits cannot exhaust
* K_rx and wedge the flow. */
f->last_active_ns = sp_ns_now_ns();
sp_ns_timer_refresh(f);
struct sp_ev_tcp_segment segment = {0};
segment.context.key.flow.value = f->flow_id;
segment.context.key.generation.value = f->core_gen;
segment.context.now.value = now;
segment.context.causal_fence.value = f->egress.next_control_seq;
segment.seq = tcp->seq;
segment.ack = tcp->ack;
segment.window = tcp->window;
segment.flags = flags;
segment.payload_length.value = (uint32_t)payload_len;
uint64_t staged_id = 0;
if (payload_len > 0) {
/* A queued asynchronous packet retains its exact TCP interval; the
* completion-priority pump ordering prevents it from overtaking the event
* that advanced the authorization frontier. */
int admit = sp_ingress_stage_admit(&g_ns_stage_pool, &f->ingress,
sp_tcp_unwrap_u32(tcp->seq,
f->local_core.rcv_acked_abs),
pkt + payload_off, payload_len,
&staged_id);
if (admit == 1)
segment.ingress.value = staged_id;
else if (admit == 0) {
/* Global/K_rx staging exhausted: drop the segment; the child retransmits.
* Pump its ACK/window/RST metadata without granting the unstaged bytes.
* FIN is sequenced after the payload and must be retried with it. */
staged_id = 0;
sp_ns_drop_unstaged_payload(&segment);
}
}
if (sp_ns_pump_start(f, sp_pump_item_segment(&segment), staged_id, 0) < 0) {
sp_ns_teardown(f);
g_ns_tunfd_for_emit = -1;
return 1;
}
sp_ns_maybe_park_timewait(f);
g_ns_tunfd_for_emit = -1;
return 1;
}
/* Backend socket readiness for a core-adapter flow. */
/* Resets the child side of a core flow and releases it, so a backend failure
* during or after connect promptly closes the child rather than leaving its
* connect()/socket to time out. */
static void sp_ns_reset_child(struct sp_ns_flow *f) {
uint32_t ack = f->local_core.rcv_acked_abs != 0 ? (uint32_t)f->local_core.rcv_acked_abs
: 0;
/* RST sequence differs by peer state. A synchronized (established) child
* accepts a RST only when its sequence equals the peer's rcv.nxt — our
* highest sent sequence, snd_nxt, per RFC 5961 (snd_una was silently
* challenged and ignored). A child still in SYN-SENT has no rcv.nxt and
* accepts on a valid ACK alone; the proven refusal form (matching the
* transitional path) uses sequence 0, which resets it immediately. */
if (g_ns_tunfd_for_emit >= 0) {
uint32_t seq = f->established ? (uint32_t)f->local_core.snd_nxt_high_abs : 0;
send_tcp_rst(g_ns_tunfd_for_emit, f->srv_ip, f->cli_ip, f->srv_port,
f->cli_port, seq, ack);
}
sp_ns_flow_release(f);
}
/* Tears down a flow after the pump returned -1. A graceful close (both FINs
* exchanged — the core reached a mutual-FIN terminal) is released quietly; every
* other termination (abort, connect failure, handshake/RTO/resource deadline,
* fatal) resets the child. The pump records which in f->pump_close_graceful. */
static void sp_ns_teardown(struct sp_ns_flow *f) {
if (f->pump_close_graceful) {
if (f->local_core.state == SP_TCP_CORE_TIME_WAIT) {
/* Preserve the quarantine even if the table is temporarily full. */
if (sp_ns_park_timewait(f) != 1)
return;
}
sp_ns_flow_release(f);
} else {
sp_ns_reset_child(f);
}
}
static void sp_ns_backend_event(struct sp_ns_flow *f, uint32_t events) {
uint64_t now = sp_ns_now_ns();
f->last_active_ns = sp_ns_now_ns();
sp_ns_timer_refresh(f);
if (((events & EPOLLERR) != 0 ||
(!f->established && (events & EPOLLHUP) != 0)) &&
(events & EPOLLIN) == 0) {
sp_ns_reset_child(f);
return;
}
/* A SOCKS proxy has a second, bounded connect phase: the proxy TCP connect
* completes first, then the SOCKS greeting/request must reach READY before
* the core sees STREAM_CONNECT_RESULT. */
if (f->use_socks && !f->established) {
if (f->cold->socks.connect_pending && (events & EPOLLOUT) != 0) {
int err = 0;
socklen_t elen = sizeof(err);
if (getsockopt(f->backend_fd, SOL_SOCKET, SO_ERROR, &err, &elen) < 0 ||
err != 0) {
sp_ns_reset_child(f);
return;
}
f->cold->socks.connect_pending = 0;
if (socks_begin_handshake(&f->cold->socks, &socks_proxy) < 0) {
sp_ns_reset_child(f);
return;
}
}
if ((events & EPOLLOUT) != 0 && socks_has_pending_tx(&f->cold->socks) &&
socks_flush_tx(f->backend_fd, &f->cold->socks) < 0) {
sp_ns_reset_child(f);
return;
}
if ((events & EPOLLIN) != 0) {
int rc = socks_recv_and_process(f->backend_fd, &f->cold->socks, &socks_proxy,
NULL);
if (rc < 0) {
sp_ns_reset_child(f);
return;
}
if (rc == 1 && f->cold->socks.state == SOCKS_IO_READY) {
struct sp_ev_connect_result result = {0};
sp_tcp_event_context_init(&result.context, f->flow_id, f->core_gen,
now);
result.effect.value = f->connect_effect_id;
result.outcome = SP_CONNECT_SUCCEEDED;
f->connect_effect_id = 0;
f->established = 1;
if (sp_ns_pump_start(f, sp_pump_item_connect_result(&result), 0, 0) < 0) {
sp_ns_teardown(f);
return;
}
/*
* A single recvmsg may contain the end of the SOCKS CONNECT reply and
* the first application bytes. The parser deliberately leaves those
* bytes in rxbuf. Publish them as the first logical backend-stream
* range before rearming EPOLLIN; otherwise they are stranded forever
* and later socket bytes would be assigned the wrong stream offsets.
*/
if (f->cold->socks.rx_len != 0) {
size_t residual_len = f->cold->socks.rx_len;
if (residual_len > sizeof(f->cold->socks_residual) ||
f->egress.stream_tail != 0 ||
f->egress.stream_consumed != 0) {
sp_ns_reset_child(f);
return;
}
memcpy(f->cold->socks_residual, f->cold->socks.rxbuf, residual_len);
f->cold->socks_residual_len = residual_len;
f->socks_residual_active = 1;
f->cold->socks.rx_len = 0;
}
}
}
if (sp_ns_update_backend_interest(f) < 0)
sp_ns_reset_child(f);
return;
}
/* Direct connect completion: the backend became writable. */
if (f->connect_effect_id != 0 && (events & EPOLLOUT) != 0) {
struct sp_ev_connect_result result = {0};
int err = 0;
int pump_rc;
socklen_t elen = sizeof(err);
if (getsockopt(f->backend_fd, SOL_SOCKET, SO_ERROR, &err, &elen) < 0 ||
err != 0) {
sp_ns_reset_child(f);
return;
}
sp_tcp_event_context_init(&result.context, f->flow_id, f->core_gen, now);
result.effect.value = f->connect_effect_id;
result.outcome = SP_CONNECT_SUCCEEDED;
f->connect_effect_id = 0;
f->established = 1;
pump_rc = sp_ns_pump_start(f, sp_pump_item_connect_result(&result), 0, 0);
if (pump_rc < 0) {
sp_ns_teardown(f);
return;
}
sp_ns_maybe_park_timewait(f);
if (f->in_use && sp_ns_update_backend_interest(f) < 0)
sp_ns_reset_child(f);
return;
}
if (!f->established)
return;
/* Backend readable: observe newly available bytes and hand the core a CREDIT
* (or STREAM_EOF). The observe path leaves authoritative bytes in the socket;
* the core will authorize a TCP_EMIT_BATCH that the pump materializes. */
if ((events & (EPOLLIN | EPOLLRDHUP | EPOLLHUP)) != 0) {
struct sp_ev_credit credit;
struct sp_ev_stream_eof eof;
int rc = sp_egress_observe_backend_typed(
&f->egress, f->backend_fd, now,
(events & (EPOLLRDHUP | EPOLLHUP)) != 0, 0,
SP_TCP_FLIGHT_ALL, SP_EFFECT_BYTES, g_ns_discard, sizeof(g_ns_discard),
g_ns_payload, sizeof(g_ns_payload), &credit, &eof);
if (rc < 0) {
DBG("observe_backend fatal (%s)", strerror(errno));
sp_ns_reset_child(f);
return;
}
/* No progress (no new backend bytes) marks the flow idle so a subsequent
* interest recompute can suppress a fruitless EPOLLIN spin; any progress
* clears it. */
f->backend_read_idle = (rc == 0);
if (rc == SP_EGRESS_OBSERVATION_CREDIT || rc == SP_EGRESS_OBSERVATION_EOF) {
int pump_rc;
/* Observe only fills stream_tail; grant the core's logical emit credit
* here, while live scheduler admission and DRR meter the physical records. */
if (rc == SP_EGRESS_OBSERVATION_CREDIT) {
credit.tun_segments.value = SP_TCP_CORE_MAX_SEGMENTS;
pump_rc = sp_ns_pump_start(f, sp_pump_item_credit(&credit), 0, 0);
} else {
pump_rc = sp_ns_pump_start(f, sp_pump_item_stream_eof(&eof), 0, 0);
}
if (pump_rc < 0) {
sp_ns_teardown(f);
return;
}
sp_ns_maybe_park_timewait(f);
}
}
/* Backend writable: re-flush any durable ingress reservation. A CREDIT with an
* unchanged stream_tail re-drives the core's [A,P) reservation reflush once
* the FIFO has drained. */
if ((events & EPOLLOUT) != 0 && f->ingress.pending_bytes > 0) {
struct sp_ev_credit credit = {0};
int pump_rc;
credit.context.key.flow.value = f->flow_id;
credit.context.key.generation.value = f->core_gen;
credit.context.now.value = now;
credit.stream_tail.value = f->egress.stream_tail;
pump_rc = sp_ns_pump_start(f, sp_pump_item_credit(&credit), 0, 0);
if (pump_rc < 0) {
sp_ns_teardown(f);
return;
}
sp_ns_maybe_park_timewait(f);
}
if (f->in_use && sp_ns_update_backend_interest(f) < 0)
sp_ns_reset_child(f);
}
/* Re-execute a flow's retained TUN emit once the output FIFO drained.
*
* A blocked emit produced no sp_ev_tun_emit_result, so the core still holds the
* matching tun_emit_effect and advanced no send-state. We therefore re-run the
* *same* effect (not a re-derived one): re-materialize the identical byte range
* (still peekable — the child has not ACKed, so nothing was consumed) and write
* it. On success the result is fed back, which advances snd-state, clears the
* core's pending effect, and may emit the next batch (itself possibly deferred
* again). On a still-full FIFO the effect stays retained for the next drain. */
static void sp_ns_retry_pending_emit(int tunfd, struct sp_ns_flow *f) {
struct sp_fx_tcp_emit_batch effect = f->pending_emit;
struct sp_ns_emit_transaction emit_txn = {
.flow = f, .effect = &effect, .tunfd = tunfd};
struct sp_ev_tun_emit_result emit_result;
uint64_t now = sp_ns_now_ns();
uint64_t saved_next_control_seq = f->egress.next_control_seq;
int preserve_scheduler_result = f->has_sched_emit_result;
int rc;
if (f->waits_sched_credit) {
struct sp_scheduler_credit credit = sp_scheduler_credit_now(&g_ns_tun_sched);
if (!sp_scheduler_credit_permits_retry(&f->sched_blocked_credit, &credit,
f->sched_blocked_dims))
return;
f->waits_sched_credit = 0;
f->sched_blocked_dims = 0;
}
g_ns_tunfd_for_emit = tunfd;
rc = sp_egress_materialize_emit(
/* A retained effect is re-materialized after a local TUN-backpressure
* interval, not at the original tail-observation point. Always take the
* explicit bounded path: retrying the third GSO batch through SO_PEEK_OFF
* reproduced the same missing-range symptom as post-consume emission. */
&f->egress, f->backend_fd, &effect, 1,
g_ns_discard, sizeof(g_ns_discard), g_ns_payload,
sizeof(g_ns_payload));
if (rc <= 0) {
if (rc < 0) {
DBG("retained-effect materialize fatal (%s)", strerror(errno));
sp_ns_teardown(f);
}
g_ns_tunfd_for_emit = -1;
return;
}
rc = sp_egress_execute_emit_with_typed(
&f->egress, sp_ns_tun_write, &emit_txn, &effect, ntohl(f->srv_ip),
ntohl(f->cli_ip), f->srv_port, f->cli_port, g_ns_payload,
(size_t)(effect.end.value - effect.start.value), 1, now, &emit_result);
if (preserve_scheduler_result) {
/* The original GSO admission already produced the one completion sequence
* held for this core effect. Scalar re-execution is write-only: preserve
* that event and roll back the executor sequence consumed by the duplicate
* completion, or the core observes a causal-sequence gap. */
f->egress.next_control_seq = saved_next_control_seq;
}
if (rc < 0) {
DBG("retained-effect emit fatal (%s)", strerror(errno));
sp_ns_teardown(f);
g_ns_tunfd_for_emit = -1;
return;
}
if (rc == 0) {
g_ns_tunfd_for_emit = -1; /* FIFO still full; keep the effect retained */
return;
}
f->has_pending_emit = 0;
memset(&f->pending_emit, 0, sizeof(f->pending_emit));
if (!preserve_scheduler_result) {
f->sched_emit_result = emit_result;
f->has_sched_emit_result = 1;
}
sp_ns_output_work_refresh(f);
g_ns_tunfd_for_emit = -1;
}
/* Re-drive every flow whose emit was FIFO-blocked, after the TUN FIFO drained.
* Each retry may itself release the flow, so re-check in_use each iteration. */
static void sp_ns_drain_pending_emits(int tunfd) {
static size_t cursor;
size_t limit = sp_ns_tcp_slot_limit();
size_t start = cursor < limit ? cursor : 0;
for (unsigned pass = 0; pass < 2; ++pass) {
size_t end = pass == 0 ? limit : start;
for (size_t slot = sp_ns_output_work_next(pass == 0 ? start : 0);
slot < end; slot = sp_ns_output_work_next(slot + 1)) {
if (g_tun_queue.count == TUN_QUEUE_PACKETS)
return;
cursor = (slot + 1) % limit;
if (g_ns_flows[slot].in_use && g_ns_flows[slot].has_pending_emit)
sp_ns_retry_pending_emit(tunfd, &g_ns_flows[slot]);
}
}
}
/* The broker's next wake-up: the earliest netstack deadline, or no deadline
* at all (UINT64_MAX) when the loop is event-driven. bounded_poll adds the
* EPOLL_TIMEOUT_MS cap for work no fd will announce: a child without a pidfd,
* or output queued for a destination that cannot be polled (O6). */
static uint64_t sp_ns_wait_deadline_ns(uint64_t now_ns, int bounded_poll) {
uint64_t deadline = UINT64_MAX;
uint64_t core_deadline = sp_deadline_next(&g_net_deadlines);
if (bounded_poll) {
uint64_t poll_ns = (uint64_t)EPOLL_TIMEOUT_MS * 1000000ULL;
deadline = UINT64_MAX - now_ns < poll_ns ? UINT64_MAX : now_ns + poll_ns;
}
if (core_deadline != 0 && core_deadline < deadline)
deadline = core_deadline;
return deadline;
}
/* Signal mask in effect only while the broker waits: the broker keeps
* SIGWINCH blocked otherwise, so a resize that arrives after the loop checked
* its flag interrupts the wait instead of being missed (O6). */
static sigset_t g_broker_wait_mask;
static int g_broker_wait_mask_set;
/* deadline_ns UINT64_MAX waits without a timeout. */
static int sp_epoll_wait_until(int epfd, struct epoll_event *events,
int maxevents, uint64_t deadline_ns,
uint64_t now_ns) {
uint64_t relative_ns = sp_tcp_ns_until(deadline_ns, now_ns);
const sigset_t *mask = g_broker_wait_mask_set ? &g_broker_wait_mask : NULL;
#ifdef __NR_epoll_pwait2
{
struct timespec timeout = {
.tv_sec = (time_t)(relative_ns / 1000000000ULL),
.tv_nsec = (long)(relative_ns % 1000000000ULL),
};
int result = (int)syscall(__NR_epoll_pwait2, epfd, events, maxevents,
deadline_ns == UINT64_MAX ? NULL : &timeout,
mask, mask != NULL ? (size_t)(_NSIG / 8) : 0);
if (result >= 0 || errno != ENOSYS)
return result;
}
#endif
{
uint64_t timeout_ms = relative_ns / 1000000ULL;
if (relative_ns % 1000000ULL != 0)
timeout_ms++;
if (timeout_ms > (uint64_t)INT_MAX)
timeout_ms = INT_MAX;
return epoll_pwait(epfd, events, maxevents,
deadline_ns == UINT64_MAX ? -1 : (int)timeout_ms, mask);
}
}
/* Fragment rejection is a packet-path diagnostic, so keep one bounded sample
* policy for both scalar and VNET records. The aggregate counter remains the
* complete evidence product; stderr is an intentionally synchronous sink. */
static uint64_t g_fragment_log_next_ns;
static void sp_log_fragment_drop(uint16_t frag_off) {
const uint64_t interval_ns = UINT64_C(1000000000);
uint64_t now_ns = sp_ns_now_ns();
if (g_fragment_log_next_ns != 0 && now_ns < g_fragment_log_next_ns)
return;
(void)sp_stderr_printf(
"[sockpuppet] IP: dropping IP fragment (frag_off=0x%04x)\n", frag_off);
g_fragment_log_next_ns =
UINT64_MAX - now_ns < interval_ns ? UINT64_MAX : now_ns + interval_ns;
}
static void dispatch_tun_ipv4_packet(int tunfd, uint8_t *pkt, size_t len) {
struct sp_packet_value parsed;
enum sp_codec_status status;
#define SP_TUN_REJECT() \
do { \
sp_counter_inc(&g_resource_counters.parser_rejects); \
return; \
} while (0)
status = sp_parse_ipv4_packet(pkt, len, 1, 1, &parsed);
if (status != SP_CODEC_OK) {
/* Keep the long-standing diagnostic for valid IPv4 fragments even though
* the shared parser rejects them before protocol dispatch. */
if (status == SP_CODEC_FRAGMENT) {
sp_counter_inc(&g_resource_counters.ipv4_fragment_drops);
sp_log_fragment_drop(parsed.ip.frag_off);
}
SP_TUN_REJECT();
}
if (!sp_policy_child_destination_allowed(&g_net_policy, &parsed))
SP_TUN_REJECT();
if (parsed.ip.protocol == IPPROTO_TCP) {
(void)handle_core_tcp(tunfd, pkt, len, &parsed);
return;
}
if (parsed.ip.protocol == IPPROTO_UDP) {
handle_udp(tunfd, pkt, len, &parsed);
return;
}
SP_TUN_REJECT();
#undef SP_TUN_REJECT
}
static void dispatch_tun_record(int tunfd, uint8_t *record, size_t len) {
struct sp_packet_value parsed;
struct sp_scalar_packet expanded[SP_TCP_GSO_POLICY_SEGS];
size_t expanded_count = 0;
enum sp_codec_status status;
status = sp_parse_tun_frame(record, len, &parsed);
if (status != SP_CODEC_OK) {
if (status == SP_CODEC_FRAGMENT) {
sp_counter_inc(&g_resource_counters.ipv4_fragment_drops);
sp_log_fragment_drop(parsed.ip.frag_off);
}
sp_counter_inc(&g_resource_counters.parser_rejects);
return;
}
if (parsed.vnet.gso_type == SP_VNET_GSO_TCPV4) {
if (sp_gso_expand_oracle(record, len, expanded,
SP_TCP_GSO_POLICY_SEGS,
&expanded_count) != SP_CODEC_OK) {
sp_counter_inc(&g_resource_counters.parser_rejects);
sp_counter_inc(&g_resource_counters.tcp_gso_expand_rejects);
return;
}
sp_counter_inc(&g_resource_counters.tcp_gso_frames_child_to_host);
for (size_t i = 0; i < expanded_count; ++i)
dispatch_tun_ipv4_packet(tunfd, expanded[i].bytes, expanded[i].len);
return;
}
uint8_t *packet = record + parsed.packet_offset;
if (sp_normalize_vnet_scalar_checksum(packet, &parsed) < 0) {
sp_counter_inc(&g_resource_counters.parser_rejects);
return;
}
dispatch_tun_ipv4_packet(tunfd, packet, parsed.ip.total_len);
}
/* Close every UDP flow whose epoll interest update failed (F2). */
static void udp_close_epoll_failed_flows(void) {
g_udp_epoll_failed = 0;
for (int i = 0; i < udp_flow_limit; ++i)
if (udp_flows[i].used && udp_flows[i].epoll_failed)
udp_close_flow(&udp_flows[i]);
}
static void udp_close_flow(struct udp_flow *f) {
struct sp_udp_key key;
uint32_t generation;
if (f == NULL)
return;
generation = f->generation;
if (f->used) {
sp_udp_outbound_key(&key, f->core.role, f->cli_ip, f->cli_port, f->srv_ip,
f->srv_port);
sp_udp_table_remove(&key, f->generation);
}
if (f->admission_held)
sp_resource_admission_release_source(f->cli_ip,
sp_udp_admission_role(f->core.role));
if (f->global_admission_held)
sp_resource_admission_release_source(0, SP_UDP_ADMISSION_GLOBAL);
(void)sp_udp_runtime_step(&f->core, SP_UDP_CLOSE, 0, 0);
while (f->core.pending_count != 0) {
uint16_t slot = f->core.pending_slots[f->core.pending_head];
udp_pending_release(slot);
sp_udp_pop(&f->core);
}
if (f->udp_relay >= 0) {
epoll_del(f->udp_relay);
close(f->udp_relay);
f->udp_relay = -1;
}
if (f->udp_staging >= 0) {
close(f->udp_staging);
f->udp_staging = -1;
}
if (f->tcp_ctrl >= 0) {
epoll_del(f->tcp_ctrl);
close(f->tcp_ctrl);
f->tcp_ctrl = -1;
}
if (f->dns_mapping_ref.generation != 0)
(void)sp_dns_core_release(&g_dns_core, f->dns_mapping_ref,
sp_ns_now_ns());
memset(f, 0, sizeof(*f));
f->generation = generation;
f->tcp_ctrl = -1;
f->udp_relay = -1;
f->udp_staging = -1;
for (size_t i = 0; i < SP_UDP_PENDING_PER_FLOW; ++i)
f->core.pending_slots[i] = UINT16_MAX;
}
/* Retire bounded due work before taking another data budget. Removing the
* owner before dispatch lets a transition reschedule itself in place. */
static void sp_net_expire_due(int tunfd, uint64_t now_ns) {
for (unsigned served = 0; served < SP_TIMER_EVENTS_TURN; ++served) {
uint64_t deadline = sp_deadline_next(&g_net_deadlines);
if (!deadline || deadline > now_ns) break;
unsigned owner = g_net_deadlines.owners[0];
if (owner >= SP_DEADLINE_TIMEWAIT) {
sp_ns_timewait_remove(&g_timewait_map.entries[owner - SP_DEADLINE_TIMEWAIT]);
continue;
}
sp_deadline_set(&g_net_deadlines, owner, 0);
if (owner >= SP_DEADLINE_PUBLISH) {
struct publish_udp_flow *f = &publish_udp_flows[owner - SP_DEADLINE_PUBLISH];
if (f->used && sp_udp_runtime_step(&f->core, SP_UDP_EXPIRE, now_ns, 0))
publish_udp_close_flow(f);
} else if (owner >= SP_DEADLINE_UDP) {
struct udp_flow *f = &udp_flows[owner - SP_DEADLINE_UDP];
if (f->used && sp_udp_runtime_step(&f->core, SP_UDP_EXPIRE, now_ns, 0))
udp_close_flow(f);
} else {
struct sp_ns_flow *f = &g_ns_flows[owner];
enum sp_tcp_timer_kind kind = SP_TCP_TIMER_HANDSHAKE;
if (!f->in_use) continue;
deadline = sp_tcp_next_deadline_ns(&f->local_core, &kind);
g_ns_tunfd_for_emit = tunfd;
if (deadline && deadline <= now_ns) {
struct sp_ev_timer_fired event = {
.context = {.key = {.flow = {f->flow_id}, .generation = {f->core_gen}},
.now = {now_ns}},
.kind = kind, .deadline = {deadline},
.timer_generation = {f->local_core.timer_gen[kind]}};
if (sp_ns_pump_start(f, sp_pump_item_timer_fired(&event), 0, 0) < 0) sp_ns_teardown(f);
} else if (!deadline) {
sp_ns_reset_child(f);
}
g_ns_tunfd_for_emit = -1;
if (f->in_use) sp_ns_timer_refresh(f);
}
}
}
static int interactive_maybe_sync_winsize(struct interactive_session *session) {
if (!session || !session->active)
return 0;
if (!interactive_resize_pending)
return 0;
interactive_resize_pending = 0;
return interactive_sync_winsize(session);
}
static int open_child_pidfd(pid_t pid) {
#ifdef __NR_pidfd_open
int pidfd = (int)syscall(__NR_pidfd_open, pid, 0);
/* pidfd_open() always sets close-on-exec. */
return pidfd;
#else
errno = ENOSYS;
return -1;
#endif
}
/* On reaping, sets *drain_deadline_ns: the CLOCK_MONOTONIC instant after
* which undelivered payload output is abandoned. */
static int broker_try_reap_child(pid_t pid, int *child_status,
uint64_t *drain_deadline_ns) {
if (*child_status != -1)
return 1;
for (;;) {
int status = 0;
pid_t got = waitpid(pid, &status, WNOHANG);
if (got == 0)
return 0;
if (got == pid) {
*child_status = status;
*drain_deadline_ns = sp_ns_now_ns() + SP_CHILD_EXIT_DRAIN_NS;
bench_mark("child_reaped", "event_loop", "ok", NULL);
return 1;
}
if (got < 0 && errno == EINTR)
continue;
DBG("waitpid(%ld, WNOHANG) failed while polling child exit: %s",
(long)pid, strerror(errno));
return -1;
}
}
static void broker_close_child_pidfd(int *child_pidfd,
int *child_pidfd_active) {
int owned;
if (child_pidfd == NULL || *child_pidfd < 0)
return;
owned = sp_take_fd(child_pidfd);
if (child_pidfd_active != NULL && *child_pidfd_active) {
if (epoll_ctl(g_epfd, EPOLL_CTL_DEL, owned, NULL) < 0 &&
errno != EBADF && errno != ENOENT)
DBG("epoll_ctl DEL child pidfd failed: %s", strerror(errno));
*child_pidfd_active = 0;
}
if (close(owned) < 0)
DBG("close child pidfd failed: %s", strerror(errno));
}
static int relay_update_epoll(struct relay_runtime *runtime) {
uint32_t source_events = EPOLLERR | EPOLLHUP | EPOLLRDHUP;
if (sp_relay_wants_read(&runtime->state))
source_events |= EPOLLIN;
if (runtime->source_fd >= 0 && !sp_relay_wants_read(&runtime->state) &&
runtime->source_registered) {
epoll_del(runtime->source_fd);
runtime->source_registered = 0;
runtime->source_interest = (struct epoll_event){0};
} else if (runtime->source_fd >= 0 &&
sp_relay_wants_read(&runtime->state)) {
struct epoll_event source_event = {
.events = source_events,
.data.u64 = epoll_registration_token(runtime->source_type, 0,
runtime->source_fd, 1)};
int operation = runtime->source_registered ? EPOLL_CTL_MOD : EPOLL_CTL_ADD;
if (sp_epoll_update(operation, runtime->source_fd, &source_event,
&runtime->source_interest) < 0)
return -1;
runtime->source_registered = 1;
}
if (runtime->destination_registered &&
sp_relay_complete(&runtime->state)) {
epoll_del(runtime->destination.fd);
runtime->destination_registered = 0;
runtime->destination_interest = (struct epoll_event){0};
} else if (runtime->destination.epollable && runtime->destination.fd >= 0 &&
!sp_relay_complete(&runtime->state)) {
uint32_t destination_events = EPOLLERR | EPOLLHUP | EPOLLRDHUP;
if (sp_relay_wants_write(&runtime->state))
destination_events |= EPOLLOUT;
struct epoll_event destination_event = {
.events = destination_events,
.data.u64 = epoll_registration_token(runtime->destination_type, 0,
runtime->destination.fd, 1)};
int operation = runtime->destination_registered ? EPOLL_CTL_MOD
: EPOLL_CTL_ADD;
if (sp_epoll_update(operation, runtime->destination.fd,
&destination_event, &runtime->destination_interest) < 0)
return -1;
runtime->destination_registered = 1;
}
return 0;
}
static int relay_matches_source(const struct relay_runtime *runtime,
const struct epoll_key *key) {
return epoll_key_matches(key, runtime->source_type, 0, runtime->source_fd, 1);
}
static int relay_matches_destination(const struct relay_runtime *runtime,
const struct epoll_key *key) {
return epoll_key_matches(key, runtime->destination_type, 0,
runtime->destination.fd, 1);
}
static int relay_active(const struct relay_runtime *runtime) {
return runtime->state.storage != NULL &&
!sp_relay_complete(&runtime->state);
}
static int interactive_update_epoll(struct relay_runtime *tty_to_pty,
struct relay_runtime *pty_to_tty) {
uint32_t tty_events = EPOLLERR | EPOLLHUP | EPOLLRDHUP;
uint32_t pty_events = EPOLLERR | EPOLLHUP | EPOLLRDHUP;
if (sp_relay_wants_read(&tty_to_pty->state))
tty_events |= EPOLLIN;