LuaJIT stack unwinding with JIT trace within it

.gitignore

libtest.so
dump.out

Makefile

LUAJIT ?= luajit
JIT ?= on

ifdef LUA_ERROR
  CPPFLAGS= -DLUA_ERROR
  LIBS= -lluajit-5.1
else
  CPPFLAGS= -ULUA_ERROR
  LIBS=
endif

all:
	g++ test.cpp -shared -fPIC $(CPPFLAGS) -olibtest.so $(LIBS)
	LUA_PATH="$(dir $(LUAJIT))?.lua;;$(LUA_PATH)" LD_LIBRARY_PATH=. \
		$(LUAJIT) -j$(shell echo $(JIT) | tr A-Z a-z)           \
		test.lua $(HOTLOOP) $(DUMP)

test.cpp

#include <lua.hpp>
#ifndef LUA_ERROR
#	include <exception>
#endif

extern "C" {

struct test {
	lua_State *L;
	int trigger;
};

void test_throw(struct test *state, int i)
{
	if (i < state->trigger)
		return;
#ifndef LUA_ERROR
	throw(new std::bad_exception());
#else
	luaL_error(state->L, "Lua error");
#endif
}

static int init(lua_State *L)
{
	struct test *state =
		(struct test *)lua_newuserdata(L, sizeof(struct test));

	state->L = L;
	state->trigger = lua_tonumber(L, 1);
	return 1;
}

LUA_API int luaopen_libtest(lua_State *L)
{
	lua_pushcfunction(L, init);
	return 1;
}

} // extern "C"

test.lua

local ffi = require('ffi')
local lib = ffi.load('libtest')
ffi.cdef('void test_throw(struct test *state, int i);')

local hotloop = arg[1] or 1
local dump = arg[2] or 'dump.out'
local raise = tonumber(hotloop) * 2
local iters = tonumber(hotloop) * 3

local test = require('libtest')(raise)

print(jit.status())
jit.opt.start('hotloop=' .. hotloop)
require('jit.dump').start('+tbisrmXaT', dump or 'dump.out')

for i = 1, iters do
	pcall(lib.test_throw, test, i)
end

print("OK")

This example succeeds with disabled JIT

$ make JIT=OFF       
g++ test.cpp -shared -fPIC -olibt.so
LUA_PATH="./?.lua;;" LD_LIBRARY_PATH=. \
	luajit -joff           \
	test.lua  
OK

However with enabled JIT it fails:

$ make JIT=ON 
g++ test.cpp -shared -fPIC -olibt.so
LUA_PATH="./?.lua;;" LD_LIBRARY_PATH=. \
	luajit -jon           \
	test.lua  
terminate called after throwing an instance of 'std::bad_exception*'
make: *** [Makefile:6: all] Aborted

Everything works fine with disabled JIT and even while trace recording:

$ make LUA_ERROR=1 JIT=OFF
g++ test.cpp -shared -fPIC -DLUA_ERROR -olibtest.so -lluajit-5.1
LUA_PATH="./?.lua;;" LD_LIBRARY_PATH=. \
        luajit -joff           \
        test.lua  
false   CMOV    SSE2    SSE3    SSE4.1  fold    cse     dce     fwd     dse     narrow  loop    abc     sink    fuse
OK
$ make LUA_ERROR=1 JIT=ON
g++ test.cpp -shared -fPIC -DLUA_ERROR -olibtest.so -lluajit-5.1
LUA_PATH="./?.lua;;" LD_LIBRARY_PATH=. \
        luajit -jon           \
        test.lua  
true    CMOV    SSE2    SSE3    SSE4.1  fold    cse     dce     fwd     dse     narrow  loop    abc     sink    fuse
OK
$ tail dump.out
0009 >  p32 HREFK  0008  "pcall" @24
0010 >  fun HLOAD  0009
0011 >  udt SLOAD  #2    T
0012 >  p32 EQ     0011  [0x400e5ef0]
0013 >  udt SLOAD  #7    T
0014 >  fun EQ     0010  pcall
0015    p64 ADD    0013  +24 
0017    nil CALLXS [0x7f016f0ef135](0015 0003)
---- TRACE 2 abort test.lua:16 -- error thrown or hook called during recording

However it fails while running the trace:

$ make LUA_ERROR=1 JIT=ON HOTLOOP=2
g++ test.cpp -shared -fPIC -DLUA_ERROR -olibtest.so -lluajit-5.1
LUA_PATH="./?.lua;;" LD_LIBRARY_PATH=. \
        luajit -jon           \
        test.lua 2 
true    CMOV    SSE2    SSE3    SSE4.1  fold    cse     dce     fwd     dse     narrow  loop    abc     sink    fuse
PANIC: unprotected error in call to Lua API (Lua error
stack traceback:
        test.lua:16: in main chunk
        [C]: at 0x55f14d4f3eb0)
make: *** [Makefile:14: all] Error 1