Rust std patch to allow thread spawn to return an error rather than panicking when out of memory for an alt signal stack

spawn-fail.patch

diff --git a/library/std/src/sys/unix/stack_overflow.rs b/library/std/src/sys/unix/stack_overflow.rs
index 1e8d1137ac8..eade97af5c6 100644
--- a/library/std/src/sys/unix/stack_overflow.rs
+++ b/library/std/src/sys/unix/stack_overflow.rs
@@ -1,28 +1,26 @@
 #![cfg_attr(test, allow(dead_code))]
 
-use self::imp::{drop_handler, make_handler};
+use crate::io;
 
-pub use self::imp::cleanup;
-pub use self::imp::init;
+use self::imp::{get_stack, install_handler};
 
+pub use self::imp::{cleanup, init};
+
+#[repr(C)]
 pub struct Handler {
-    data: *mut libc::c_void,
+    stack: *mut libc::stack_t,
 }
 
 impl Handler {
-    pub unsafe fn new() -> Handler {
-        make_handler()
+    pub unsafe fn new() -> io::Result<Handler> {
+        Ok( Handler { stack: get_stack()? } )
     }
 
-    fn null() -> Handler {
-        Handler { data: crate::ptr::null_mut() }
-    }
-}
-
-impl Drop for Handler {
-    fn drop(&mut self) {
-        unsafe {
-            drop_handler(self.data);
+    pub unsafe fn install(self) {
+        if !self.stack.is_null() {
+            install_handler(self.stack);
+            let stack = Box::from_raw(self.stack);
+            drop(stack);
         }
     }
 }
@@ -38,7 +36,6 @@ fn drop(&mut self) {
     target_os = "openbsd"
 ))]
 mod imp {
-    use super::Handler;
     use crate::io;
     use crate::mem;
     use crate::ptr;
@@ -50,7 +47,7 @@ mod imp {
     use libc::{sigaltstack, SIGSTKSZ, SS_DISABLE};
     use libc::{MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE, SIGSEGV};
 
-    use crate::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
+    use crate::sync::atomic::{AtomicPtr, Ordering};
     use crate::sys::unix::os::page_size;
     use crate::sys_common::thread_info;
 
@@ -117,11 +114,11 @@ unsafe fn siginfo_si_addr(info: *mut libc::siginfo_t) -> usize {
         }
     }
 
-    static MAIN_ALTSTACK: AtomicPtr<libc::c_void> = AtomicPtr::new(ptr::null_mut());
-    static NEED_ALTSTACK: AtomicBool = AtomicBool::new(false);
+    static MAIN_ALTSTACK: AtomicPtr<libc::stack_t> = AtomicPtr::new(ptr::null_mut() as *mut _);
 
     pub unsafe fn init() {
         let mut action: sigaction = mem::zeroed();
+        let mut need_altstack = false;
         for &signal in &[SIGSEGV, SIGBUS] {
             sigaction(signal, ptr::null_mut(), &mut action);
             // Configure our signal handler if one is not already set.
@@ -129,20 +126,34 @@ pub unsafe fn init() {
                 action.sa_flags = SA_SIGINFO | SA_ONSTACK;
                 action.sa_sigaction = signal_handler as sighandler_t;
                 sigaction(signal, &action, ptr::null_mut());
-                NEED_ALTSTACK.store(true, Ordering::Relaxed);
+                need_altstack = true;
             }
         }
+        if !need_altstack {
+            return;
+        }
 
-        let handler = make_handler();
-        MAIN_ALTSTACK.store(handler.data, Ordering::Relaxed);
-        mem::forget(handler);
+        let stack = match get_stack() {
+            Ok(stack) => stack,
+            Err(e) => {
+                rtprintpanic!(
+                    "\n{}: could not create signal stack: {}\n",
+                    thread::current().name().unwrap_or("<unknown>"),
+                    e,
+                );
+                rtabort!("create signal stack");
+            }
+        };
+        install_handler(stack);
+        MAIN_ALTSTACK.store(stack, Ordering::Relaxed);
     }
 
     pub unsafe fn cleanup() {
-        drop_handler(MAIN_ALTSTACK.load(Ordering::Relaxed));
+        let altstack = MAIN_ALTSTACK.swap(ptr::null_mut(), Ordering::Release);
+        uninstall_stack(altstack);
     }
 
-    unsafe fn get_stackp() -> *mut libc::c_void {
+    unsafe fn get_stackp() -> io::Result<*mut libc::c_void> {
         // OpenBSD requires this flag for stack mapping
         // otherwise the said mapping will fail as a no-op on most systems
         // and has a different meaning on FreeBSD
@@ -153,37 +164,36 @@ unsafe fn get_stackp() -> *mut libc::c_void {
         let stackp =
             mmap(ptr::null_mut(), SIGSTKSZ + page_size(), PROT_READ | PROT_WRITE, flags, -1, 0);
         if stackp == MAP_FAILED {
-            panic!("failed to allocate an alternative stack: {}", io::Error::last_os_error());
+            return Err(io::Error::last_os_error());
         }
         let guard_result = libc::mprotect(stackp, page_size(), PROT_NONE);
         if guard_result != 0 {
-            panic!("failed to set up alternative stack guard page: {}", io::Error::last_os_error());
+            return Err(io::Error::last_os_error());
         }
-        stackp.add(page_size())
+        Ok(stackp.add(page_size()))
     }
 
-    unsafe fn get_stack() -> libc::stack_t {
-        libc::stack_t { ss_sp: get_stackp(), ss_flags: 0, ss_size: SIGSTKSZ }
+    pub unsafe fn get_stack() -> io::Result<*mut libc::stack_t> {
+        let stack = libc::stack_t { ss_sp: get_stackp()?, ss_flags: 0, ss_size: SIGSTKSZ };
+        Ok(Box::into_raw(box stack))
     }
 
-    pub unsafe fn make_handler() -> Handler {
-        if !NEED_ALTSTACK.load(Ordering::Relaxed) {
-            return Handler::null();
-        }
-        let mut stack = mem::zeroed();
-        sigaltstack(ptr::null(), &mut stack);
-        // Configure alternate signal stack, if one is not already set.
-        if stack.ss_flags & SS_DISABLE != 0 {
-            stack = get_stack();
-            sigaltstack(&stack, ptr::null_mut());
-            Handler { data: stack.ss_sp as *mut libc::c_void }
-        } else {
-            Handler::null()
+    pub unsafe fn install_handler(stack: *mut libc::stack_t) {
+        let mut old_stack = mem::zeroed();
+        sigaltstack(ptr::null(), &mut old_stack);
+        if old_stack.ss_flags & SS_DISABLE == 0 {
+            // Will not replace enabled signal stack.
+            assert!(!stack.is_null());
+            let stack = Box::from_raw(stack);
+            drop(stack);
+            return;
         }
+        // Configure alternate signal stack, since one is not already enabled.
+        sigaltstack(stack, ptr::null_mut());
     }
 
-    pub unsafe fn drop_handler(data: *mut libc::c_void) {
-        if !data.is_null() {
+    unsafe fn uninstall_stack(old_stack: *mut libc::stack_t) {
+        if !old_stack.is_null() {
             let stack = libc::stack_t {
                 ss_sp: ptr::null_mut(),
                 ss_flags: SS_DISABLE,
@@ -194,9 +204,14 @@ pub unsafe fn drop_handler(data: *mut libc::c_void) {
                 ss_size: SIGSTKSZ,
             };
             sigaltstack(&stack, ptr::null_mut());
-            // We know from `get_stackp` that the alternate stack we installed is part of a mapping
-            // that started one page earlier, so walk back a page and unmap from there.
-            munmap(data.sub(page_size()), SIGSTKSZ + page_size());
+            if (*old_stack).ss_flags & SS_DISABLE == 0 {
+                // We know from `get_stackp` that the alternate stack we installed is part of a mapping
+                // that started one page earlier, so walk back a page and unmap from there.
+
+                assert_eq!(0, munmap((*old_stack).ss_sp.sub(page_size()), SIGSTKSZ + page_size()));
+                let old_stack = Box::from_raw(old_stack);
+                drop(old_stack);
+            }
         }
     }
 }
@@ -216,9 +231,11 @@ pub unsafe fn init() {}
 
     pub unsafe fn cleanup() {}
 
-    pub unsafe fn make_handler() -> super::Handler {
-        super::Handler::null()
+    pub unsafe fn get_stack() -> io::Result<*mut libc::stack_t> {
+        Ok(libc::crate::ptr::null_mut())
     }
 
-    pub unsafe fn drop_handler(_data: *mut libc::c_void) {}
+    pub unsafe fn install_handler(_stack: *mut libc::stack_t) {}
+
+    pub unsafe fn uninstall_stack() {}
 }
diff --git a/library/std/src/sys/unix/thread.rs b/library/std/src/sys/unix/thread.rs
index 9e02966b57c..8a27964e0e8 100644
--- a/library/std/src/sys/unix/thread.rs
+++ b/library/std/src/sys/unix/thread.rs
@@ -46,10 +46,16 @@ pub struct Thread {
 unsafe impl Send for Thread {}
 unsafe impl Sync for Thread {}
 
+#[repr(C)]
+struct TArgs {
+    handler: stack_overflow::Handler,
+    p: *mut Box<dyn FnOnce()>,
+}
+
 impl Thread {
     // unsafe: see thread::Builder::spawn_unchecked for safety requirements
     pub unsafe fn new(stack: usize, p: Box<dyn FnOnce()>) -> io::Result<Thread> {
-        let p = Box::into_raw(box p);
+        let p: *mut Box<dyn FnOnce()> = Box::into_raw(box p);
         let mut native: libc::pthread_t = mem::zeroed();
         let mut attr: libc::pthread_attr_t = mem::zeroed();
         assert_eq!(libc::pthread_attr_init(&mut attr), 0);
@@ -84,7 +90,14 @@ pub unsafe fn new(stack: usize, p: Box<dyn FnOnce()>) -> io::Result<Thread> {
             };
         }
 
-        let ret = libc::pthread_create(&mut native, &attr, thread_start, p as *mut _);
+        // Set up our stack overflow handler which may get triggered if we run
+        // out of stack.
+        let handler = stack_overflow::Handler::new()?;
+        let targs: *mut TArgs = Box::into_raw(box TArgs {
+            handler,
+            p,
+        });
+        let ret = libc::pthread_create(&mut native, &attr, thread_start, targs as *mut _);
         // Note: if the thread creation fails and this assert fails, then p will
         // be leaked. However, an alternative design could cause double-free
         // which is clearly worse.
@@ -99,13 +112,17 @@ pub unsafe fn new(stack: usize, p: Box<dyn FnOnce()>) -> io::Result<Thread> {
             Ok(Thread { id: native })
         };
 
-        extern "C" fn thread_start(main: *mut libc::c_void) -> *mut libc::c_void {
+        extern "C" fn thread_start(targs: *mut libc::c_void) -> *mut libc::c_void {
             unsafe {
-                // Next, set up our stack overflow handler which may get triggered if we run
-                // out of stack.
-                let _handler = stack_overflow::Handler::new();
+                // Take the arguments apart.
+                let targs: Box<TArgs> = Box::from_raw(targs as *mut TArgs);
+                // Move the stack overflow signal handler and code onto the
+                // current stack.
+                let handler = targs.handler;
+                let p = Box::from_raw(targs.p);
+                handler.install();
                 // Finally, let's run some code.
-                Box::from_raw(main as *mut Box<dyn FnOnce()>)();
+                (*p)();
             }
             ptr::null_mut()
         }