epoll difference from illumos and linux

00-readme.md

See the block comment in eventfd-test.c for more information:

cc eventfd-test.c -lpthread -o eventfd-test
./eventfd-test

Example output on hardware virtalized Linux:

$ ./eventfd-test
iterations: 5
read-after-write: false
[thread1]: (0) wrote data to event_fd
[thread2]: (0) got epoll events
[thread1]: (1) wrote data to event_fd
[thread2]: (1) got epoll events
[thread1]: (2) wrote data to event_fd
[thread2]: (2) got epoll events
[thread1]: (3) wrote data to event_fd
[thread2]: (3) got epoll events
[thread1]: (4) wrote data to event_fd
[thread2]: (4) got epoll events
[thread2]: finished
[thread1]: finished

Example output on LX (software virtualized) Linux on illumos:

$ ./eventfd-test
iterations: 5
read-after-write: false
[thread1]: (0) wrote data to event_fd
[thread2]: (0) got epoll events
[thread1]: (1) wrote data to event_fd
... hangs ...

eventfd-test.c

/*
 * Simple test program to exercise possible LX bug wrt epoll/event2fd.
 *
 * This program works be creating an epoll fd and eventfd fd.  The eventfd fd is
 * registered to be watched by the epoll fd, and then 2 threads are spun up to
 * communicate with eachother via the eventfd.
 *
 * Thread1:
 *   - writes to the event fd
 *   - waits for a condition signal to continue
 *   - loop
 * Thread2:
 *   - waits for epoll events on the event fd
 *   - (optional) reads the data from the event fd
 *   - signals thread 1 to wakeup
 *   - loops
 *
 * The optional step of reading the data in Thread2 is what can trigger the bug.
 * If the data is read, this program works without issue.  If the data is not
 * read, epoll will only generate an event for the first write to the eventfd,
 * but not subsequent writes.
 *
 * There are 2 #define's that can modify this programs behavior
 *   - NUM_ITERATIONS - number of times to loop for each thread, should be >=2
 *                      to exercise this bug.
 *   - READ_EVENT_FD - whether to read the data in Thread2 (the optional step),
 *                     setting this to `false` will exercise the bug and at this
 *                     current moment result in a hang after the 2nd call to
 *                     write.
 *
 * More info on bug origin:
 * https://gist.github.com/bahamas10/497acef27b8519e824e90d9eb648a6f6
 *
 * Author: Dave Eddy <dave@daveeddy.com>
 * Date: March 19, 2022
 * License: MIT
 */

#include <err.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>

#define READ_EVENT_FD false
#define NUM_ITERATIONS 5

// shared with threads
struct context {
        int epoll_fd;
        int event_fd;
        pthread_mutex_t *lock;
        pthread_cond_t *cond;
};

void write_to_event_fd(int event_fd) {
        static uint64_t data = 1;

        if (write(event_fd, &data, sizeof (data)) != sizeof (data)) {
                err(1, "write to event_fd");
        }
}

void read_from_event_fd(int event_fd) {
        static uint64_t want_data = 1;
        uint64_t have_data;

        if (read(event_fd, &have_data, sizeof (have_data)) != sizeof (have_data)) {
                err(1, "read from event_fd");
        }

        if (want_data != have_data) {
                err(1, "read bad data from event_fd");
        }
}

void wait_for_epoll_events(int epoll_fd) {
        int nevents = 50;
        int timeout = -1;
        struct epoll_event ep_events[nevents];
        int num_events;

        num_events = epoll_wait(epoll_fd, ep_events, nevents, timeout);

        if (num_events != 1) {
                err(1, "epoll_wait returned bad value: %d", num_events);
        }
}

void *thread1_worker(void *ptr) {
        struct context *ctx = ptr;

        for (int i = 0; i < NUM_ITERATIONS; i++) {
                pthread_mutex_lock(ctx->lock);

                write_to_event_fd(ctx->event_fd);
                printf("[thread1]: (%d) wrote data to event_fd\n", i);

                pthread_cond_wait(ctx->cond, ctx->lock);
                pthread_mutex_unlock(ctx->lock);
        }

        printf("[thread1]: finished\n");
}

void *thread2_worker(void *ptr) {
        struct context *ctx = ptr;

        for (int i = 0; i < NUM_ITERATIONS; i++) {
                wait_for_epoll_events(ctx->epoll_fd);
                printf("[thread2]: (%d) got epoll events\n", i);

                pthread_mutex_lock(ctx->lock);

                if (READ_EVENT_FD) {
                        read_from_event_fd(ctx->event_fd);
                        printf("[thread2]: (%d) read from event_fd\n", i);
                }

                pthread_cond_signal(ctx->cond);
                pthread_mutex_unlock(ctx->lock);
        }

        printf("[thread2]: finished\n");
}

int main(int arg, char **argv) {
        int ret;
        struct epoll_event ev;
        pthread_t thread1, thread2;
        pthread_mutex_t lock;
        pthread_cond_t cond;

        printf("iterations: %d\n", NUM_ITERATIONS);
        printf("read-after-write: %s\n", READ_EVENT_FD ? "true" : "false");

        // create mutex
        if (pthread_mutex_init(&lock, NULL) != 0) {
                err(1, "pthread_mutex_init");
        }

        // create cond variable
        if (pthread_cond_init(&cond, NULL) != 0) {
                err(1, "pthread_cond_init");
        }

        // create epoll handle
        int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
        if (epoll_fd == -1) {
                err(1, "epoll_create1");
        }

        // create eventfd handle
        int event_fd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
        if (event_fd == -1) {
                err(1, "eventfd create");
        }

        // add eventfd handle to epoll event watcher
        ev.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
        ev.data.ptr = NULL;
        ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &ev);
        if (ret == -1) {
                err(1, "epoll_ctl (add eventfd to epoll)");
        }

        struct context ctx = {
                epoll_fd,
                event_fd,
                &lock,
                &cond
        };

        // create threads to test communication
        pthread_create(&thread1, NULL, thread1_worker, &ctx);
        pthread_create(&thread2, NULL, thread2_worker, &ctx);

        pthread_join(thread1, NULL);
        pthread_join(thread2, NULL);

        return 0;
}

@papertigers has simplified this code to not need threads -> https://gist.github.com/papertigers/5ae99002a2428d04ca736c0522ca43fe