tmp_memfd.py

import ctypes
import os
import time


def _sendfile(trgt_fd, src_fd):
    # Cf. https://github.com/python/cpython/blob/3.8/Lib/shutil.py#L114
    # which we cannot easily use directly.
    try:
        blocksize = max(os.fstat(src_fd).st_size, 2 ** 23)  # min 8MiB
    except OSError:
        blocksize = 2 ** 27  # 128MiB

    offset = 0
    while True:
        try:
            sent = os.sendfile(trgt_fd, src_fd, offset, blocksize)
        except OSError:
            raise
        else:
            if sent == 0:
                break  # EOF
            offset += sent


def new_dl():
    src_fd = os.open("/opt/nle/build/libnethack.so", os.O_RDONLY)
    try:
        trgt_fd = os.memfd_create("nle.so")
        _sendfile(trgt_fd, src_fd)
        return trgt_fd, "/proc/self/fd/%i" % trgt_fd
    finally:
        os.close(src_fd)


def main():
    fd1, dlpath1 = new_dl()
    fd2, dlpath2 = new_dl()

    print(dlpath1, dlpath2)

    dl1 = ctypes.CDLL(dlpath1, mode=ctypes.RTLD_LOCAL)
    dl2 = ctypes.CDLL(dlpath2, mode=ctypes.RTLD_LOCAL)

    stat1 = os.fstat(fd1)
    stat2 = os.fstat(fd2)

    assert stat1.st_dev == stat2.st_dev
    assert stat1.st_ino != stat2.st_ino

    # int locknum; is some NH global.
    ctypes.c_int.in_dll(dl1, "locknum").value = 42  # Set in dl1.
    assert ctypes.c_int.in_dll(dl2, "locknum").value == 0  # dl2 unchanged.
    ctypes.c_int.in_dll(dl2, "locknum").value = 69  # Set in dl2.
    assert ctypes.c_int.in_dll(dl1, "locknum").value == 42  # dl1 unchanged.


if __name__ == "__main__":
    main()

tmp_memfs_mac.py

import ctypes
import os
import shutil
import tempfile

import posix  # macOS _fcopyfile


def _sendfile(trgt_fd, src_fd):
    # Cf. https://github.com/python/cpython/blob/3.8/Lib/shutil.py#L114
    # which we cannot easily use directly.
    try:
        blocksize = max(os.fstat(src_fd).st_size, 2 ** 23)  # min 8MiB
    except OSError:
        blocksize = 2 ** 27  # 128MiB

    offset = 0
    while True:
        sent = os.sendfile(trgt_fd, src_fd, offset, blocksize)
        if sent == 0:
            break  # EOF
        offset += sent


def new_dl():
    # Auto-closed file: Create and remove from file system.
    fd, path = tempfile.mkstemp(suffix="nle.so")
    os.unlink(path)  # Unfortunately, this doesn't work.

    # This causes:
    #     self._handle = _dlopen(self._name, mode)
    # OSError: dlopen(/dev/fd/4, 6): no suitable image found.  Did find:
    # 	/dev/fd/4: code signature in (/dev/fd/4) not valid for use in process using Library Validation: library load disallowed by system policy

    # Apparently the system checks if st_nlink == 0?

    with open("/Users/hnr/src/nle/build/libnethack.so", "rb") as src:
        print("src", os.fstat(src.fileno()))
        while True:
            buf = src.read(shutil.COPY_BUFSIZE)
            if not buf:
                break
            os.write(fd, buf)
        # posix._fcopyfile(src.fileno(), fd, posix._COPYFILE_DATA)

    os.lseek(fd, 0, os.SEEK_SET)

    return fd, "/dev/fd/%i" % fd


def main():
    fd1, dlpath1 = new_dl()
    fd2, dlpath2 = new_dl()

    print(dlpath1, os.fstat(fd1))
    print(dlpath2, os.fstat(fd2))

    # print(os.listdir("/dev/fd"))

    # f = os.open(dlpath1, os.O_RDONLY)
    # print(os.read(f, 32))

    dl1 = ctypes.CDLL(dlpath1, mode=ctypes.RTLD_LOCAL)
    dl2 = ctypes.CDLL(dlpath2, mode=ctypes.RTLD_LOCAL)

    stat1 = os.fstat(fd1)
    stat2 = os.fstat(fd2)

    assert stat1.st_dev == stat2.st_dev
    assert stat1.st_ino != stat2.st_ino

    # int locknum; is some NH global.
    ctypes.c_int.in_dll(dl1, "locknum").value = 42  # Set in dl1.
    assert ctypes.c_int.in_dll(dl2, "locknum").value == 0  # dl2 unchanged.
    ctypes.c_int.in_dll(dl2, "locknum").value = 69  # Set in dl2.
    assert ctypes.c_int.in_dll(dl1, "locknum").value == 42  # dl1 unchanged.


if __name__ == "__main__":
    main()