netns shared umem

README.md

Build

git clone https://github.com/xdp-project/xdp-tools.git --recurse-submodules
make -C xdp-tools
ln -s ./xdp-tools/lib/libxdp/xsk_def_xdp_prog.o
make

Setup network interfaces

ip netns add test-ns
ip link add outer-veth type veth peer inner-veth netns test-ns

ethtool -K outer-veth tx off rxvlan off txvlan off
ip link set dev outer-veth up
ip addr add dev outer-veth 10.0.0.1/30

ip netns exec test-ns /bin/bash <<EOF
  ip link set dev lo up

  ethtool -K inner-veth tx off rxvlan off txvlan off
  ip link set dev inner-veth up
  ip addr add dev inner-veth 10.0.0.2/30
EOF

Run the binary

sudo ./netns-shared-umem

Generate packets from root namespace

ping 10.0.0.2

Makefile

USER_TARGETS := netns-shared-umem

LDFLAGS += -pthread
LIB_DIR = xdp-tools/lib

include $(LIB_DIR)/common.mk

netns-shared-umem.c

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sched.h>
#include <linux/if_link.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <poll.h>

#include <bpf/xsk.h>
#include <bpf/libbpf.h>
#include <xdp/libxdp.h>

#define SZ_FRAME 2048
#define UMEM_NR_DESCS 2048

struct worker_args
{
    struct xsk_umem *xsk_umem;
};

void *worker(struct worker_args *args)
{
    // switch netns
    int netns_fd;
    netns_fd = open("/run/netns/test-ns", O_RDONLY);
    if (netns_fd == -1)
    {
        fprintf(stderr, "error opening netns");
        exit(EXIT_FAILURE);
    }
    if (setns(netns_fd, 0) == -1)
    {
        fprintf(stderr, "error entering netns");
        exit(EXIT_FAILURE);
    }

    struct xsk_socket *xsk = NULL;
    struct xsk_socket_config xsk_config = {
        .rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
        .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
        .libbpf_flags = 0, // load default xdp program from libbpf (xsk_def_prog)
        .xdp_flags = XDP_FLAGS_DRV_MODE,
        .bind_flags = XDP_USE_NEED_WAKEUP, // | XDP_ZEROCOPY,
    };
    struct xsk_ring_cons rx_ring = {0};
    struct xsk_ring_prod tx_ring = {0};
    struct xsk_ring_cons comp_ring = {0};
    struct xsk_ring_prod fill_ring = {0};
    int errno;
    if ((errno = xsk_socket__create_shared(&xsk,
                                           "inner-veth",
                                           0, args->xsk_umem,
                                           &rx_ring,
                                           &tx_ring,
                                           &fill_ring,
                                           &comp_ring,
                                           &xsk_config)) != 0)
    {
        fprintf(stderr, "xsk_socket__create_shared error: %s\n", strerror(-errno));
        exit(EXIT_FAILURE);
    }

    printf("inner xsk created: %p\n", xsk);

    // add all descriptors to the fill ring
    unsigned int fill_ring_pos = 0;
    xsk_ring_prod__reserve(&fill_ring, XSK_RING_PROD__DEFAULT_NUM_DESCS, &fill_ring_pos);
    for (int i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
        *xsk_ring_prod__fill_addr(&fill_ring, fill_ring_pos + i) = i * SZ_FRAME;
    xsk_ring_prod__submit(&fill_ring, XSK_RING_PROD__DEFAULT_NUM_DESCS);

    unsigned int rx_ring_pos = 0;
    // lets read one packet
    do
    {
        unsigned int n_pkts = xsk_ring_cons__peek(&rx_ring, 1, &rx_ring_pos);
        if (n_pkts != 1)
        {
            if (xsk_ring_prod__needs_wakeup(&fill_ring))
            {
                struct pollfd pollfd = {
                    .fd = xsk_socket__fd(xsk),
                    .events = POLLIN,
                };
                poll(&pollfd, 1, 0);
            }
        }
        else
        {
            break;
        }
    } while (1);

    const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&rx_ring, rx_ring_pos);

    printf("got a descriptor with addr: %lld and len %d\n", desc->addr, desc->len);

    exit(EXIT_SUCCESS);
}

int main()
{
    int errno = 0;
    struct xsk_umem *xsk_umem = NULL;
    void *umem_area = NULL;
    struct xsk_umem_config umem_config = {
        .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
        .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
        .frame_size = SZ_FRAME,
        .frame_headroom = 0,
        .flags = 0,
    };
    struct xsk_ring_prod fill_ring = {0};
    struct xsk_ring_cons completion_ring = {0};

    if ((errno = posix_memalign(&umem_area, getpagesize(), SZ_FRAME * UMEM_NR_DESCS)) != 0)
    {
        fprintf(stderr, "posix_memalign error: %s\n", strerror(-errno));
        return EXIT_FAILURE;
    }

    if ((errno = xsk_umem__create(&xsk_umem, umem_area, SZ_FRAME * UMEM_NR_DESCS, &fill_ring, &completion_ring, &umem_config)) != 0)
    {
        fprintf(stderr, "xsk_umem__create: %s\n", strerror(-errno));
        return EXIT_FAILURE;
    }

    printf("umem created, mem_area: %p, len: %d\n", umem_area, SZ_FRAME * UMEM_NR_DESCS);

    struct worker_args wa = {
        .xsk_umem = xsk_umem,
    };

    pthread_t inner_thread;
    pthread_create(&inner_thread, NULL, (void *(*)(void *))worker, &wa);

    pthread_join(inner_thread, NULL);

    return 0;
}