bad stack

gistfile1.txt

I added the following two probes to lx_emulate_user()

--- a/usr/src/uts/intel/brand/lx/lx_archdep.c
+++ b/usr/src/uts/intel/brand/lx/lx_archdep.c
@@ -1438,6 +1438,8 @@ lx_emulate_user(klwp_t *lwp, int syscall_num, uintptr_t *a
                 */
                ucontext_t uc;
 
+DTRACE_PROBE5(lx__callstack, int, syscall_num, void *, lwpd, void *, sp,
+    void *, top, int, frsz);
                /*
                 * We do not want to save the signal mask for an emulation
                 * context.  Some emulated system calls alter the signal mask;
@@ -1447,6 +1449,8 @@ lx_emulate_user(klwp_t *lwp, int syscall_num, uintptr_t *a
                savecontext(&uc, NULL);
 
                if (on_fault(&lab)) {
+DTRACE_PROBE3(lx__badstack, int, syscall_num, proc_t *, p, void *, sp);
                        goto badstack;
                }

------------------------------------

I wrote a dtrace script that fires these probes and checks the addresses
we're trying to access.

This is the key portion of the trace for the thread that gets the bad stack:
126136 9 -> brk
126136 9 <- brk
126136 9 -> mmap
126136 9 lx-callstack sc 9 sp 7ffffc4aec68 nstk 7ffffc4af000 nsc 7ffffc4af000 tp 7ffffc4af000 sz 920
126136 9 ERR invalid address 7ffffc4af000
126136 9 ERR invalid address 7ffffc4af000
126136 9 ERR invalid address 7ffffc4af000
126136 9 JJ pgN 7ffffc4aeffc OK
126136 9 JJ sp 7ffffc4aec68 OK
126136 9 <- mmap
126136 9 send signal: 10 to: java pid 126136, tid 9
126136 9 handle signal: 10
126136 9 -> rt_sigprocmask
126136 9 lx-callstack sc 14 sp 7ffffc4add58 nstk 7ffffc4af000 nsc 7ffffc4ae178 tp 7ffffc4ae0f0 sz 920
126136 9 JJ nsc 7ffffc4ae178 OK
126136 9 JJ top 7ffffc4ae0f0 OK
126136 9 JJ pg1 7ffffc4ae000 OK
126136 9 ERR invalid address 7ffffc4adffc
126136 9 ERR invalid address 7ffffc4add58
126136 9 lx-badstack sc 14 sp 7ffffc4add58

-----------------------------------

This is the dtrace script that produced that output:

ERROR
{
    printf("%d %d ERR %s %p\n", pid, tid,
         (arg4 == 1 ? "invalid address" : "other"), arg5);
}

proc:::signal-send
/execname == "java"/
{
        printf("%d %d send signal: %d to: %s pid %d, tid %d\n", pid, tid,
            args[2], stringof(args[1]->pr_fname), args[1]->pr_pid,
            args[0]->pr_lwpid);
}

proc:::signal-handle
/execname == "java"/
{
        printf("%d %d handle signal: %d\n", pid, tid, args[0]);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->l = (lx_lwp_data_t *)arg1;
        printf("%d %d lx-callstack sc %d sp %p nstk %p nsc %p tp %p sz %d\n",
           pid, tid, (int)arg0, arg2, this->l->br_ntv_stack,
           this->l->br_ntv_stack_current, arg3, arg4);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->l = (lx_lwp_data_t *)arg1;
        this->uv = (int *)copyin(this->l->br_ntv_stack_current, 4);
        printf("%d %d JJ nsc %p OK\n",
           pid, tid, this->l->br_ntv_stack_current);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->uv = (int *)copyin(arg3, 4);
        printf("%d %d JJ top %p OK\n", pid, tid, arg3);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->a = arg3 & ~0xfff;
        this->uv = (int *)copyin(this->a, 4);
        printf("%d %d JJ pg1 %p OK\n", pid, tid, this->a);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->a = arg3 & ~0xfff;
        this->a = this->a - 4;
        this->uv = (int *)copyin(this->a, 4);
        printf("%d %d JJ pgN %p OK\n", pid, tid, this->a);
}

sdt:::lx-callstack
/execname == "java"/
{
        this->uv = (int *)copyin(arg2, 4);
        printf("%d %d JJ sp %p OK\n", pid, tid, arg2);
}

sdt:::lx-badstack
{
        printf("%d %d lx-badstack sc %d sp %p\n", pid, tid, (int)arg0,
           arg2);
}

lx-syscall:::entry
/execname == "java"/
{
        printf("%d %d -> %s\n", pid, tid, probefunc);
}

lx-syscall:::return
/execname == "java"/
{
        printf("%d %d <- %s\n", pid, tid, probefunc);
}