uring statx failure

README

Stating a bunch of files in a git repo fails in a weird way.

It doesn't fail every time but in my checkout of https://github.com/git/git it fails about 30% of the time.

I invoke:
  git ls-files | strace -o /tmp/stat-many.strace ~/Projects/git-dirty/stat-many $PWD
  
I have attached a failing strace.

I'm running a 5.7.17 kernel on amd64 with liburing 0.7 from Debian.

------

I attached an even simpler test case that just stats itself.

simpler.c

#include <ctype.h>
#include <fcntl.h>
#include <linux/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include <liburing.h>

#define URING_ENTRIES 1024
#define STAT_TOTAL (128 * 1024)
#define STAT_BATCH 256

int main(int argc, const char **argv) {
  // Open the current directory.
  char dir[PATH_MAX];
  int dirfd;
  getcwd(dir, PATH_MAX);
  dirfd = open(dir, O_RDONLY | O_DIRECTORY);
  if (dirfd < 0) {
    perror("open directory");
    exit(-1);
  }

  // Initialize the uring.
  struct io_uring ring;
  io_uring_queue_init(URING_ENTRIES, &ring, 0);

  // Stat this file ~STAT_TOTAL times in STAT_BATCH chunks.
  struct statx *stats = calloc(STAT_BATCH, sizeof(struct statx));
  for (size_t first = 0; first < STAT_TOTAL; first += STAT_BATCH) {
    // Queue up the stats.
    for (size_t i = 0; i < STAT_BATCH; i++) {
      struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
      io_uring_prep_statx(sqe, dirfd, argv[0], 0,
                          STATX_TYPE | STATX_MODE | STATX_UID | STATX_GID |
                              STATX_MTIME | STATX_CTIME | STATX_INO |
                              STATX_SIZE,
                          &stats[i]);
    }

    // Submit the requests.
    int ret = io_uring_submit(&ring);
    if (ret < 0) {
      fprintf(stderr, "io_uring_submit failed: %s\n", strerror(ret));
      exit(ret);
    }

    // Read the responses.
    for (size_t i = 0; i < STAT_BATCH; i++) {
      struct io_uring_cqe *cqe = NULL;
      if ((ret = io_uring_wait_cqe(&ring, &cqe))) {
        fprintf(stderr, "io_uring_wait_cqe failed: %s\n", strerror(ret));
        exit(ret);
      }
      io_uring_cqe_seen(&ring, cqe);
    }
  }
  free(stats);
}

stat-many.c

#include <ctype.h>
#include <fcntl.h>
#include <linux/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>

#include <liburing.h>

#define MAX_FILES (128 * 1024)
#define URING_ENTRIES 1024
#define STAT_BATCH 256

int dirfd;
char **filenames;
struct io_uring ring;

void open_dir(const char *path) {
  dirfd = open(path, O_RDONLY | O_DIRECTORY);
  if (dirfd < 0) {
    perror("open directory");
    exit(-1);
  }
}

size_t read_filenames() {
  filenames = malloc(MAX_FILES * sizeof(char **));
  size_t i = 0;
  while (i < MAX_FILES - 1 && !feof(stdin)) {
    size_t n = 128;
    char *line = NULL;
    ssize_t s = getline(&line, &n, stdin);
    if (s < 0) {
      if (feof(stdin)) {
        break;
      }
      perror("readline");
      exit(1);
    }
    while (s > 0 && isspace(line[--s])) {
      line[s] = '\0';
    }
    filenames[i++] = line;
  }
  filenames[i] = NULL;
  return i;
}

void uring_stat(size_t first, size_t count) {
  struct statx *stats = calloc(count, sizeof(struct statx));

  for (size_t i = 0; i < count; i++) {
    struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
    const char *filename = filenames[first + i];
    io_uring_prep_statx(sqe, dirfd, filename, 0,
                        STATX_TYPE | STATX_MODE | STATX_UID | STATX_GID |
                            STATX_MTIME | STATX_CTIME | STATX_INO | STATX_SIZE,
                        &stats[i]);
    io_uring_sqe_set_data(sqe, (void *)filename);
  }

  int ret = io_uring_submit(&ring);
  if (ret < 0) {
    fprintf(stderr, "io_uring_submit failed: %s\n", strerror(ret));
    exit(ret);
  }

  for (size_t i = 0; i < count; i++) {
    struct io_uring_cqe *cqe = NULL;
    if ((ret = io_uring_wait_cqe(&ring, &cqe))) {
      fprintf(stderr, "io_uring_wait_cqe failed: %s\n", strerror(ret));
      exit(ret);
    }
    io_uring_cqe_seen(&ring, cqe);
  }
  
  free(stats);
}

int main(int argc, const char **argv) {
  if (argc != 2) {
    fprintf(stderr, "%s <dir>\n", argv[0]);
    exit(1);
  }

  open_dir(argv[1]);

  size_t num_filenames = read_filenames();

  io_uring_queue_init(URING_ENTRIES, &ring, 0);

  for (size_t first = 0; first < num_filenames; first += STAT_BATCH) {
    size_t count = num_filenames - first;
    if (count > STAT_BATCH) {
      count = STAT_BATCH;
    }
    uring_stat(first, count);
  }
}

stat-many.strace

execve("/home/ianloic/Projects/git-dirty/stat-many", ["/home/ianloic/Projects/git-dirty"..., "/home/ianloic/git/git"], 0x7ffc9096c308 /* 70 vars */) = 0
brk(NULL)                               = 0xc12000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=129347, ...}) = 0
mmap(NULL, 129347, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f95c058f000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/liburing.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\21\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14184, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f95c058d000
mmap(NULL, 16400, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f95c0588000
mmap(0x7f95c0589000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f95c0589000
mmap(0x7f95c058a000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f95c058a000
mmap(0x7f95c058b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f95c058b000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\240\t\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1870824, ...}) = 0
mmap(NULL, 1886208, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f95c03bb000
mmap(0x7f95c0451000, 901120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x96000) = 0x7f95c0451000
mmap(0x7f95c052d000, 303104, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x172000) = 0x7f95c052d000
mmap(0x7f95c0577000, 57344, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7f95c0577000
mmap(0x7f95c0585000, 10240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f95c0585000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\362\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1321344, ...}) = 0
mmap(NULL, 1323280, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f95c0277000
mmap(0x7f95c0286000, 630784, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x7f95c0286000
mmap(0x7f95c0320000, 626688, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa9000) = 0x7f95c0320000
mmap(0x7f95c03b9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x141000) = 0x7f95c03b9000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0203\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=100736, ...}) = 0
mmap(NULL, 103496, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f95c025d000
mmap(0x7f95c0260000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f95c0260000
mmap(0x7f95c0271000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x7f95c0271000
mmap(0x7f95c0275000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f95c0275000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@n\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1839792, ...}) = 0
mmap(NULL, 1852680, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f95c0098000
mprotect(0x7f95c00bd000, 1662976, PROT_NONE) = 0
mmap(0x7f95c00bd000, 1355776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7f95c00bd000
mmap(0x7f95c0208000, 303104, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x170000) = 0x7f95c0208000
mmap(0x7f95c0253000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ba000) = 0x7f95c0253000
mmap(0x7f95c0259000, 13576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f95c0259000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f95c0096000
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f95c0093000
arch_prctl(ARCH_SET_FS, 0x7f95c0093740) = 0
mprotect(0x7f95c0253000, 12288, PROT_READ) = 0
mprotect(0x7f95c0275000, 4096, PROT_READ) = 0
mprotect(0x7f95c03b9000, 4096, PROT_READ) = 0
mprotect(0x7f95c0577000, 45056, PROT_READ) = 0
mprotect(0x7f95c058b000, 4096, PROT_READ) = 0
mprotect(0x403000, 4096, PROT_READ)     = 0
mprotect(0x7f95c05d9000, 4096, PROT_READ) = 0
munmap(0x7f95c058f000, 129347)          = 0
brk(NULL)                               = 0xc12000
brk(0xc33000)                           = 0xc33000
openat(AT_FDCWD, "/home/ianloic/git/git", O_RDONLY|O_DIRECTORY) = 3
mmap(NULL, 1052672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f95bff92000
fstat(0, {st_mode=S_IFIFO|0600, st_size=0, ...}) = 0
read(0, ".cirrus.yml\n.clang-format\n.edito"..., 4096) = 4096
read(0, "ocumentation/RelNotes/1.7.1.1.tx"..., 4096) = 4096
read(0, "Documentation/RelNotes/2.0.2.txt"..., 4096) = 4096
read(0, "entation/RelNotes/2.4.0.txt\nDocu"..., 4096) = 4096
brk(0xc54000)                           = 0xc54000
read(0, "status.txt\nDocumentation/config/"..., 4096) = 4096
read(0, "tation/git-remote-ext.txt\nDocume"..., 4096) = 4096
read(0, "entation/technical/api-merge.txt"..., 4096) = 4096
read(0, "\nbuiltin/upload-archive.c\nbuilti"..., 4096) = 4096
read(0, "ntrib/credential/netrc/Makefile\n"..., 4096) = 4096
read(0, "e.c\ndecorate.h\ndelta-islands.c\nd"..., 4096) = 4096
read(0, "send.c\niterator.h\njson-writer.c\n"..., 4096) = 4096
brk(0xc75000)                           = 0xc75000
read(0, "00compile.t\nt/Git-SVN/Utils/add_"..., 4096) = 4096
read(0, "st-path-utils.c\nt/helper/test-pk"..., 4096) = 4096
read(0, "60-path-utils.sh\nt/t0061-run-com"..., 4096) = 4096
read(0, "01-update-index-reupdate.sh\nt/t2"..., 4096) = 4096
read(0, "c.sh\nt/t4003-diff-rename-1.sh\nt/"..., 4096) = 4096
read(0, "4013/diff.diff_--dirstat_initial"..., 4096) = 4096
read(0, "anged_-SF_master\nt/t4013/diff.no"..., 4096) = 4096
brk(0xc96000)                           = 0xc96000
read(0, "s-definition\nt/t4018/matlab-func"..., 4096) = 4096
read(0, "apply-multifrag.sh\nt/t4109/expec"..., 4096) = 4096
read(0, "t5100/info0001\nt/t5100/info0002\n"..., 4096) = 4096
read(0, "t-0036-report-multi-rewrite-for-"..., 4096) = 4096
read(0, "15/refs.br-config-explicit-octop"..., 4096) = 4096
read(0, "e-http.sh\nt/t5813-proto-disable-"..., 4096) = 4096
read(0, "t-msg-hook.sh\nt/t7505-prepare-co"..., 4096) = 4096
brk(0xcb7000)                           = 0xcb7000
read(0, "n-fetch-merge.sh\nt/t9158-git-svn"..., 4096) = 4096
read(0, "plates/this--description\nthread-"..., 4096) = 1357
read(0, "", 4096)                       = 0
io_uring_setup(1024, {flags=0, sq_thread_cpu=0, sq_thread_idle=0, sq_entries=1024, cq_entries=2048, features=IORING_FEAT_SINGLE_MMAP|IORING_FEAT_NODROP|IORING_FEAT_SUBMIT_STABLE|IORING_FEAT_RW_CUR_POS|IORING_FEAT_CUR_PERSONALITY|0x20, sq_off={head=0, tail=64, ring_mask=256, ring_entries=264, flags=276, dropped=272, array=33088}, cq_off={head=128, tail=192, ring_mask=260, ring_entries=268, overflow=280, cqes=320}}) = 4
mmap(NULL, 37184, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_POPULATE, 4, 0) = 0x7f95c05a5000
mmap(NULL, 65536, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_POPULATE, 4, 0x10000000) = 0x7f95c0595000
io_uring_enter(4, 256, 0, 0, NULL, 8)   = 256
io_uring_enter(4, 256, 0, 0, NULL, 8)   = 256
io_uring_enter(4, 0, 1, IORING_ENTER_GETEVENTS, NULL, 8) = 0
io_uring_enter(4, 0, 1, IORING_ENTER_GETEVENTS, NULL, 8) = -1 EBADF (Bad file descriptor)
write(2, "io_uring_wait_cqe failed: Unknow"..., 43) = -1 EBADF (Bad file descriptor)
exit_group(-9)                          = ?
+++ exited with 247 +++