test-nowait.c

/// Test for RWF_NOWAIT, does it always non-block?
/// The answer is, it may sometimes, for example when there is some issues with the underlying disk.
/// Or maybe when it accessing page entries, that are blocked for a long time, need to take a look deeper.

#define _GNU_SOURCE /// for preadv2()
#include <sys/uio.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>

int main(int argc, const char ** argv)
{
    int fd;
    int err;
    int exitcode = 0;
    struct stat st;

    if (argc != 2)
        return 1;

    fd = open(argv[1], O_RDONLY);
    if (fd < 0)
        perror("open");

    err = fstat(fd, &st);
    if (err)
        perror("fstat");

    size_t i;
    size_t offset;
    size_t errors[4096] = {0};
    for (i = 0, offset = 0; offset < st.st_size; ++i, offset += 4096)
    {
        ssize_t ret;
        struct timespec start;
        struct timespec end;
        char buffer[4096];
        struct iovec iov;

        iov.iov_base = buffer;
        iov.iov_len = 4096;

        clock_gettime(CLOCK_MONOTONIC, &start);
        ret = preadv2(fd, &iov, 1, offset, RWF_NOWAIT);
        if (ret < 0)
            ++errors[errno];
        clock_gettime(CLOCK_MONOTONIC, &end);

        {
            size_t start_nsec = start.tv_sec*1e9 + start.tv_nsec;
            size_t end_nsec = end.tv_sec*1e9 + end.tv_nsec;
            size_t diff = end_nsec - start_nsec;
            if (diff > 1e9)
            {
                printf("too long at offset=%zu, ret=%zu (%.4f)\n", i, ret, diff/1e9);
                exitcode = 1;
            }
        }
    }

    for (size_t j = 0; j < 4096; ++j)
        if (errors[j])
            printf("%s: %zu (out of %zu)\n", strerrorname_np(j), errors[j], i);

    return exitcode;
}

$ while yes /path/to/big/file | head -n 100 | xargs -P100 -n1 /tmp/test-nowait ; do :; done
...
EAGAIN: 8565670 (out of 17148317)
EAGAIN: 8565670 (out of 17148317)
EAGAIN: 8565670 (out of 17148317)
too long at offset=8691694, ret=18446744073709551615 (14.5016)
too long at offset=8694766, ret=18446744073709551615 (14.4338)
too long at offset=8694766, ret=18446744073709551615 (16.2144)
...

I also had this in parallel just in case:

$ dd if=/dev/sda of=/dev/null bs=32M status=progress iflag=direct
$ while :; do time pv /path/to/big/file > /dev/null; time dd oflag=nocache conv=notrunc,fdatasync count=0 of=/path/to/big/file; done

# uname -r
4.19.0-20-amd64