check_mmap_jit_behaviour.c

#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <sys/mman.h>

void print_prot_bits(int prot) {
  printf((prot & PROT_READ) == 0 ? "-" : "R");
  printf((prot & PROT_WRITE) == 0 ? "-" : "W");
  printf((prot & PROT_EXEC) == 0 ? "-" : "X");
}

void *try_mmap_jit(size_t size, int prot) {
  int map_flags = MAP_ANON | MAP_PRIVATE | MAP_JIT;
  printf("Try mmap with MAP_JIT: ");
  print_prot_bits(prot);
  void *mem = mmap(NULL, size, prot, map_flags, -1, 0);
  if (mem == MAP_FAILED || mem == NULL) {
    printf(" FAIL: %s", strerror(errno));
    return NULL;
  }
  printf(" PASS: %p", mem);
  return mem;
}

void try_mprotect(void *mem, size_t size, int prot) {
  printf("Try mprotect: %p ", mem);
  print_prot_bits(prot);
  if (mprotect(mem, size, prot) != 0)
    printf(" FAIL: %s", strerror(errno));
  else
    printf(" PASS");
}

void try_mmap_jit_and_mprotect(size_t size, int prot_mmap, int prot_mprotect) {
  void *addr = try_mmap_jit(size, prot_mmap);
  printf(" -> ");
  if (addr) {
    try_mprotect(addr, size, prot_mprotect);
  }
  putchar('\n');
}

int main(int argc, const char *argv[]) {
  for (int prot_mmap = 0; prot_mmap < 8; prot_mmap++) {
    for (int prot_mprotect = 0; prot_mprotect < 8; prot_mprotect++) {
      try_mmap_jit_and_mprotect(4096, prot_mmap, prot_mprotect);
    }
  }
  return 0;
}

result_on_11_2.txt

$ sw_vers -productVersion
11.2

$ uname -mprsv
Darwin 20.3.0 Darwin Kernel Version 20.3.0: Thu Jan 21 00:06:51 PST 2021; root:xnu-7195.81.3~1/RELEASE_ARM64_T8101 arm64 arm

$ clang -o check_mmap_jit_behaviour check_mmap_jit_behaviour.c && ./check_mmap_jit_behaviour
Try mmap with MAP_JIT: --- PASS: 0x1000e0000 -> Try mprotect: 0x1000e0000 --- PASS
Try mmap with MAP_JIT: --- PASS: 0x1000e4000 -> Try mprotect: 0x1000e4000 R-- PASS
Try mmap with MAP_JIT: --- PASS: 0x1000e8000 -> Try mprotect: 0x1000e8000 -W- PASS
Try mmap with MAP_JIT: --- PASS: 0x1000ec000 -> Try mprotect: 0x1000ec000 RW- PASS
Try mmap with MAP_JIT: --- PASS: 0x1000f0000 -> Try mprotect: 0x1000f0000 --X PASS
Try mmap with MAP_JIT: --- PASS: 0x1000f4000 -> Try mprotect: 0x1000f4000 R-X PASS
Try mmap with MAP_JIT: --- PASS: 0x1000f8000 -> Try mprotect: 0x1000f8000 -WX PASS
Try mmap with MAP_JIT: --- PASS: 0x1000fc000 -> Try mprotect: 0x1000fc000 RWX PASS
Try mmap with MAP_JIT: R-- PASS: 0x100100000 -> Try mprotect: 0x100100000 --- PASS
Try mmap with MAP_JIT: R-- PASS: 0x100104000 -> Try mprotect: 0x100104000 R-- PASS
Try mmap with MAP_JIT: R-- PASS: 0x100108000 -> Try mprotect: 0x100108000 -W- PASS
Try mmap with MAP_JIT: R-- PASS: 0x10010c000 -> Try mprotect: 0x10010c000 RW- PASS
Try mmap with MAP_JIT: R-- PASS: 0x100110000 -> Try mprotect: 0x100110000 --X PASS
Try mmap with MAP_JIT: R-- PASS: 0x100114000 -> Try mprotect: 0x100114000 R-X PASS
Try mmap with MAP_JIT: R-- PASS: 0x100118000 -> Try mprotect: 0x100118000 -WX PASS
Try mmap with MAP_JIT: R-- PASS: 0x10011c000 -> Try mprotect: 0x10011c000 RWX PASS
Try mmap with MAP_JIT: -W- PASS: 0x100120000 -> Try mprotect: 0x100120000 --- PASS
Try mmap with MAP_JIT: -W- PASS: 0x100124000 -> Try mprotect: 0x100124000 R-- PASS
Try mmap with MAP_JIT: -W- PASS: 0x100128000 -> Try mprotect: 0x100128000 -W- PASS
Try mmap with MAP_JIT: -W- PASS: 0x10012c000 -> Try mprotect: 0x10012c000 RW- PASS
Try mmap with MAP_JIT: -W- PASS: 0x100130000 -> Try mprotect: 0x100130000 --X PASS
Try mmap with MAP_JIT: -W- PASS: 0x100134000 -> Try mprotect: 0x100134000 R-X PASS
Try mmap with MAP_JIT: -W- PASS: 0x100138000 -> Try mprotect: 0x100138000 -WX PASS
Try mmap with MAP_JIT: -W- PASS: 0x10013c000 -> Try mprotect: 0x10013c000 RWX PASS
Try mmap with MAP_JIT: RW- PASS: 0x100140000 -> Try mprotect: 0x100140000 --- PASS
Try mmap with MAP_JIT: RW- PASS: 0x100144000 -> Try mprotect: 0x100144000 R-- PASS
Try mmap with MAP_JIT: RW- PASS: 0x100148000 -> Try mprotect: 0x100148000 -W- PASS
Try mmap with MAP_JIT: RW- PASS: 0x10014c000 -> Try mprotect: 0x10014c000 RW- PASS
Try mmap with MAP_JIT: RW- PASS: 0x100150000 -> Try mprotect: 0x100150000 --X PASS
Try mmap with MAP_JIT: RW- PASS: 0x100154000 -> Try mprotect: 0x100154000 R-X PASS
Try mmap with MAP_JIT: RW- PASS: 0x100158000 -> Try mprotect: 0x100158000 -WX PASS
Try mmap with MAP_JIT: RW- PASS: 0x10015c000 -> Try mprotect: 0x10015c000 RWX PASS
Try mmap with MAP_JIT: --X PASS: 0x100160000 -> Try mprotect: 0x100160000 --- PASS
Try mmap with MAP_JIT: --X PASS: 0x100164000 -> Try mprotect: 0x100164000 R-- PASS
Try mmap with MAP_JIT: --X PASS: 0x100168000 -> Try mprotect: 0x100168000 -W- PASS
Try mmap with MAP_JIT: --X PASS: 0x10016c000 -> Try mprotect: 0x10016c000 RW- PASS
Try mmap with MAP_JIT: --X PASS: 0x100170000 -> Try mprotect: 0x100170000 --X PASS
Try mmap with MAP_JIT: --X PASS: 0x100174000 -> Try mprotect: 0x100174000 R-X PASS
Try mmap with MAP_JIT: --X PASS: 0x100178000 -> Try mprotect: 0x100178000 -WX PASS
Try mmap with MAP_JIT: --X PASS: 0x10017c000 -> Try mprotect: 0x10017c000 RWX PASS
Try mmap with MAP_JIT: R-X PASS: 0x100180000 -> Try mprotect: 0x100180000 --- PASS
Try mmap with MAP_JIT: R-X PASS: 0x100184000 -> Try mprotect: 0x100184000 R-- PASS
Try mmap with MAP_JIT: R-X PASS: 0x100188000 -> Try mprotect: 0x100188000 -W- PASS
Try mmap with MAP_JIT: R-X PASS: 0x10018c000 -> Try mprotect: 0x10018c000 RW- PASS
Try mmap with MAP_JIT: R-X PASS: 0x100190000 -> Try mprotect: 0x100190000 --X PASS
Try mmap with MAP_JIT: R-X PASS: 0x100194000 -> Try mprotect: 0x100194000 R-X PASS
Try mmap with MAP_JIT: R-X PASS: 0x100198000 -> Try mprotect: 0x100198000 -WX PASS
Try mmap with MAP_JIT: R-X PASS: 0x10019c000 -> Try mprotect: 0x10019c000 RWX PASS
Try mmap with MAP_JIT: -WX PASS: 0x1001a0000 -> Try mprotect: 0x1001a0000 --- FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001a4000 -> Try mprotect: 0x1001a4000 R-- FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001a8000 -> Try mprotect: 0x1001a8000 -W- FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001ac000 -> Try mprotect: 0x1001ac000 RW- FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001b0000 -> Try mprotect: 0x1001b0000 --X FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001b4000 -> Try mprotect: 0x1001b4000 R-X FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001b8000 -> Try mprotect: 0x1001b8000 -WX FAIL: Permission denied
Try mmap with MAP_JIT: -WX PASS: 0x1001bc000 -> Try mprotect: 0x1001bc000 RWX FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001c0000 -> Try mprotect: 0x1001c0000 --- FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001c4000 -> Try mprotect: 0x1001c4000 R-- FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001c8000 -> Try mprotect: 0x1001c8000 -W- FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001cc000 -> Try mprotect: 0x1001cc000 RW- FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001d0000 -> Try mprotect: 0x1001d0000 --X FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001d4000 -> Try mprotect: 0x1001d4000 R-X FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001d8000 -> Try mprotect: 0x1001d8000 -WX FAIL: Permission denied
Try mmap with MAP_JIT: RWX PASS: 0x1001dc000 -> Try mprotect: 0x1001dc000 RWX FAIL: Permission denied

Thanks for the test, it clearly shows the essence of the bug. mprotect() with any permissions doesn't work after MAP_JIT region receives WX permissions (regardless if the permissions were initial or were obtained via an earlier mprotect() call).

Have you filed a ticket to Apple? If you didn't, don't you mind if I use your snippet to do that?

@roolebo
Not yet. (Actually, I'm not sure how to submit a ticket to Apple).
Sure! Feel free to use my snippet for a report. Please post the link to a ticket filed here.

Great, thanks! The ticket is: FB8994773

FWIW. You can submit bugs via feedback assistant app or https://feedbackassistant.apple.com. More details on the process:
https://developer.apple.com/bug-reporting/