Lua 5.4.0-rc5 low memory segfault reproduction files

gdb-backtrace-when-compiled-with-O0

Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7cdef98 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0  0x00007ffff7cdef98 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007ffff7ce1c98 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#2  0x00007ffff7ce2f25 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3  0x00007ffff7ce3f8f in realloc () from /lib/x86_64-linux-gnu/libc.so.6
#4  0x000055555557f73e in l_alloc (ud=0x0, ptr=0x5555555a28c0, osize=544, nsize=1536) at lauxlib.c:1009
#5  0x000055555556974d in luaM_realloc_ (L=0x5555555a2268, block=0x5555555a28c0, osize=544, nsize=1536) at lmem.c:166
#6  0x0000555555562b47 in luaD_reallocstack (L=0x5555555a2268, newsize=96, raiseerror=1) at ldo.c:187
#7  0x0000555555562cf6 in luaD_growstack (L=0x5555555a2268, n=85, raiseerror=1) at ldo.c:231
#8  0x00005555555639d3 in luaD_call (L=0x5555555a2268, func=0x5555555a2910, nresults=-1) at ldo.c:494
#9  0x0000555555563b75 in luaD_callnoyield (L=0x5555555a2268, func=0x5555555a2910, nResults=-1) at ldo.c:526
#10 0x000055555555f544 in f_call (L=0x5555555a2268, ud=0x7fffffffdd80) at lapi.c:997
#11 0x00005555555629be in luaD_rawrunprotected (L=0x5555555a2268, f=0x55555555f50f <f_call>, ud=0x7fffffffdd80) at ldo.c:148
#12 0x0000555555564316 in luaD_pcall (L=0x5555555a2268, func=0x55555555f50f <f_call>, u=0x7fffffffdd80, old_top=80, ef=64) at ldo.c:749
#13 0x000055555555f614 in lua_pcallk (L=0x5555555a2268, nargs=0, nresults=-1, errfunc=3, ctx=0, k=0x0) at lapi.c:1023
#14 0x000055555555ba60 in docall (L=0x5555555a2268, narg=0, nres=-1) at lua.c:139
#15 0x000055555555be5d in handle_script (L=0x5555555a2268, argv=0x7fffffffe260) at lua.c:228
#16 0x000055555555ca08 in pmain (L=0x5555555a2268) at lua.c:603
#17 0x00005555555638d5 in luaD_call (L=0x5555555a2268, func=0x5555555a28d0, nresults=1) at ldo.c:482
#18 0x0000555555563b75 in luaD_callnoyield (L=0x5555555a2268, func=0x5555555a28d0, nResults=1) at ldo.c:526
#19 0x000055555555f544 in f_call (L=0x5555555a2268, ud=0x7fffffffe110) at lapi.c:997
#20 0x00005555555629be in luaD_rawrunprotected (L=0x5555555a2268, f=0x55555555f50f <f_call>, ud=0x7fffffffe110) at ldo.c:148
#21 0x0000555555564316 in luaD_pcall (L=0x5555555a2268, func=0x55555555f50f <f_call>, u=0x7fffffffe110, old_top=16, ef=0) at ldo.c:749
#22 0x000055555555f614 in lua_pcallk (L=0x5555555a2268, nargs=2, nresults=1, errfunc=0, ctx=0, k=0x0) at lapi.c:1023
#23 0x000055555555cb2d in main (argc=2, argv=0x7fffffffe258) at lua.c:629

test-script.lua

return function()
    return "str0",
    function()
        return "str1",
        function()
            return "str2",
            function()
            end,"str3",
            function()
            end,"str4",
            function()
            end,"str5",
            function()
            end
        end,"str6",
        function()
            return "str7",
            function()
            end,"str8",
            function()
            end,"str9",
            function()
            end,"str10",
            function()
            end
        end,"str11",
        function()
            return "str12",
            function()
            end,"str13",
            function()
            end,"str14",
            function()
            end,"str15",
            function()
            end
        end,"str16",
        function()
        end
    end,"str17",
    function()
        return "str18",
        function()
            return "str19",
            function()
            end,"str20",
            function()
            end,"str21",
            function()
            end,"str22",
            function()
            end
        end,"str23",
        function()
            return "str24",
            function()
            end,"str25",
            function()
            end,"str26",
            function()
            end,"str27",
            function()
            end
        end,"str28",
        function()
            return "str29",
            function()
            end,"str30",
            function()
            end,"str31",
            function()
            end,"str32",
            function()
            end
        end,"str33",
        function()
        end

    end,"str34",
    function()
        return "str35",
        function()
            return "str36",
            function()
            end,"str37",
            function()
            end,"str38",
            function()
            end,"str39",
            function()
            end
        end,"str40",
        function()
            return "str41",
            function()
            end,"str42",
            function()
            end,"str43",
            function()
            end,"str44",
            function()
            end
        end,"str45",
        function()
            return "str46",
            function()
            end,"str47",
            function()
            end,"str48",
            function()
            end,"str49",
            function()
            end
        end,"str50",
        function()
        end

    end,"str51",
    function()
        return "str52",
        function()
            return "str53",
            function()
            end,"str54",
            function()
            end,"str55",
            function()
            end,"str56",
            function()
            end
        end,"str57",
        function()
            return "str58",
            function()
            end,"str59",
            function()
            end,"str60",
            function()
            end,"str61",
            function()
            end
        end,"str62",
        function()
            return "str63",
            function()
            end,"str64",
            function()
            end,"str65",
            function()
            end,"str66",
            function()
            end
        end,"str67",
        function()
        end
    end
end

traced-failing-alloc.patch

diff --git a/src/lauxlib.c b/src/lauxlib.c
index e3d9be3..81b2814 100644
--- a/src/lauxlib.c
+++ b/src/lauxlib.c
@@ -982,15 +982,35 @@ LUALIB_API const char *luaL_gsub (lua_State *L, const char *s,
   return lua_tostring(L, -1);
 }
 
-
+static int alloc_idx = 0;
+static int fail_alloc_at = 345 /* found by brute force, likely dependent on test script */;
 static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) {
+  void* ret;
+  char failed = 0;
   (void)ud; (void)osize;  /* not used */
-  if (nsize == 0) {
-    free(ptr);
-    return NULL;
+  alloc_idx++;
+  if (alloc_idx == fail_alloc_at)
+  {
+    if (nsize <= osize)
+      fprintf(stderr, "fail_alloc_at: %08d: cannot fail\n", fail_alloc_at);
+    else {
+      fprintf(stderr, "fail_alloc_at: %08d: failing\n", fail_alloc_at);
+      failed = 1;
+      ret = NULL;
+    }
   }
-  else
-    return realloc(ptr, nsize);
+  if (!failed)
+  {
+    if (nsize == 0) {
+      free(ptr);
+      ret = NULL;
+    }
+    else {
+      ret = realloc(ptr, nsize);
+    }
+  }
+  fprintf(stderr, "%08d %p %u %p %u\n", alloc_idx, ptr, (unsigned)osize, ret, (unsigned)nsize);
+  return ret;
 }