mono 3.2.8 ThreadPriority patch

From 53d170e17ad6fd46f2d551e2df25277db6e4d4cf Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey <justincc@justincc.org>
Date: Tue, 16 Sep 2014 18:03:42 +0100
Subject: [PATCH] This is a modified patch from
 https://bugzilla.novell.com/show_bug.cgi?id=540524 updated to build against
 mono 3.2.8

This uses Thread.Priority to change RT thread priority if mono is run via "$ sudo chrt --rr 10 <exec>" or similar
For many reasons this is not a good approach - this patch is primarily for test purposes.
---
 mcs/class/corlib/System.Threading/Thread.cs |  10 +-
 mono/io-layer/threads.h                     |   2 +
 mono/io-layer/wthreads.c                    | 219 ++++++++++++++++++++++++++++
 mono/metadata/icall-def.h                   |   2 +
 mono/metadata/threads-types.h               |   2 +
 mono/metadata/threads.c                     |  37 +++++
 6 files changed, 270 insertions(+), 2 deletions(-)

diff --git a/mcs/class/corlib/System.Threading/Thread.cs b/mcs/class/corlib/System.Threading/Thread.cs
index 5af6800..446af1f 100644
--- a/mcs/class/corlib/System.Threading/Thread.cs
+++ b/mcs/class/corlib/System.Threading/Thread.cs
@@ -585,13 +585,19 @@ namespace System.Threading {
 			}
 		}
 
+    [MethodImplAttribute(MethodImplOptions.InternalCall)]
+    private extern static ThreadPriority GetPriority_internal (InternalThread thread);
+ 
+    [MethodImplAttribute(MethodImplOptions.InternalCall)]
+    private extern static void SetPriority_internal (InternalThread thread, ThreadPriority priority);
+
 		public ThreadPriority Priority {
 			get {
-				return(ThreadPriority.Lowest);
+        return GetPriority_internal (Internal);
 			}
 			
 			set {
-				// FIXME: Implement setter.
+        SetPriority_internal (Internal, value);
 			}
 		}
 
diff --git a/mono/io-layer/threads.h b/mono/io-layer/threads.h
index 12b7c0c..65ec7d8 100644
--- a/mono/io-layer/threads.h
+++ b/mono/io-layer/threads.h
@@ -59,6 +59,8 @@ extern void Sleep(guint32 ms);
 extern guint32 SleepEx(guint32 ms, gboolean alertable);
 extern guint32 QueueUserAPC (WapiApcProc apc_callback, gpointer thread_handle, 
 					gpointer param);
+extern gint32 GetThreadPriority (gpointer handle);
+extern gboolean SetThreadPriority (gpointer handle, gint32 priority);
 
 /* Kludge alert! Making this visible outside io-layer is broken, but I
  * can't find any w32 call that will let me do this.
diff --git a/mono/io-layer/wthreads.c b/mono/io-layer/wthreads.c
index e623d71..0bcffcd 100644
--- a/mono/io-layer/wthreads.c
+++ b/mono/io-layer/wthreads.c
@@ -70,6 +70,14 @@ struct _WapiHandleOps _wapi_thread_ops = {
 	NULL				/* prewait */
 };
 
+typedef enum {
+	THREAD_PRIORITY_LOWEST = -2,
+	THREAD_PRIORITY_BELOW_NORMAL = -1,
+	THREAD_PRIORITY_NORMAL = 0,
+	THREAD_PRIORITY_ABOVE_NORMAL = 1,
+	THREAD_PRIORITY_HIGHEST = 2
+} WapiThreadPriority;
+
 static mono_once_t thread_ops_once=MONO_ONCE_INIT;
 
 static void thread_ops_init (void)
@@ -1306,3 +1314,214 @@ void _wapi_thread_disown_mutex (gpointer mutex)
 	
 	g_ptr_array_remove (thread_handle->owned_mutexes, mutex);
 }
+
+/**
+ * _wapi_thread_posix_priority_to_priority:
+ *
+ *   Convert a POSIX priority to a WapiThreadPriority.
+ * sched_priority is a POSIX priority,
+ * policy is the current scheduling policy
+ */
+static WapiThreadPriority _wapi_thread_posix_priority_to_priority (int sched_priority, int policy)
+{
+/* Necessary to get valid priority range */
+#ifdef _POSIX_PRIORITY_SCHEDULING
+	int max,
+	    min,
+	    i,
+	    priority,
+	    chunk;
+	WapiThreadPriority priorities[] = {
+		THREAD_PRIORITY_LOWEST,
+		THREAD_PRIORITY_LOWEST,
+		THREAD_PRIORITY_BELOW_NORMAL,
+		THREAD_PRIORITY_NORMAL,
+		THREAD_PRIORITY_ABOVE_NORMAL,
+		THREAD_PRIORITY_HIGHEST,
+		THREAD_PRIORITY_HIGHEST
+	};
+	    
+	max = sched_get_priority_max (policy);
+	min = sched_get_priority_min (policy);
+	
+	/* Partition priority range linearly, 
+	   assign each partition a thread priority */
+	if (max != min && 0 <= max && 0 <= min) {
+		for (i=1, priority=min, chunk=(max-min)/7; 
+		     i<6 && sched_priority > priority;
+		     ++i) {
+			priority += chunk;
+		}
+		
+		if (max <= priority)
+    {
+      g_debug("Get Using priority %i", THREAD_PRIORITY_HIGHEST);
+			return (THREAD_PRIORITY_HIGHEST);
+    }
+		else
+    {
+      g_debug("Get Using priority %i", priorities[i-1]);
+			return (priorities[i-1]);
+    }
+	}
+#endif
+
+	return (THREAD_PRIORITY_NORMAL);
+}
+
+/**
+ * _wapi_thread_priority_to_posix_priority:
+ *
+ *   Convert a WapiThreadPriority to a POSIX priority.
+ * priority is a WapiThreadPriority,
+ * policy is the current scheduling policy
+ */
+static int _wapi_thread_priority_to_posix_priority (WapiThreadPriority priority, int policy)
+{
+/* Necessary to get valid priority range */
+#ifdef _POSIX_PRIORITY_SCHEDULING
+	int max,
+	    min,
+	    posix_priority,
+	    i;
+	WapiThreadPriority priorities[] = {
+		THREAD_PRIORITY_LOWEST,
+		THREAD_PRIORITY_LOWEST,
+		THREAD_PRIORITY_BELOW_NORMAL,
+		THREAD_PRIORITY_NORMAL,
+		THREAD_PRIORITY_ABOVE_NORMAL,
+		THREAD_PRIORITY_HIGHEST,
+		THREAD_PRIORITY_HIGHEST
+	};
+	
+	max = sched_get_priority_max (policy);
+	min = sched_get_priority_min (policy);
+
+      g_debug("WAPI input in Set %i", priority);
+      g_debug("Min in Set %i", min);
+      g_debug("Max in Set %i", max);
+      g_debug("Policy in Set %i", policy);
+	
+	/* Partition priority range linearly, 
+	   numerically approximate matching ThreadPriority */
+	if (max != min && 0 <= max && 0 <= min) {
+		for (i=0; i<7; ++i) {
+			if (priorities[i] == priority) {
+				posix_priority = min + ((max-min)/7) * i;
+				if (max < posix_priority)
+        {
+      g_debug("Set Using max priority %i", max);
+					return max;
+        }
+				else {
+      g_debug("Set Using priority %i",posix_priority);
+					return posix_priority;
+				}
+			}
+		}
+	}
+#endif
+
+	switch (policy) {
+		case SCHED_FIFO:
+		case SCHED_RR:
+			return 50;
+		case SCHED_BATCH:
+		case SCHED_OTHER:
+			return 0;
+		default:
+			return -1;
+	}
+}
+
+/**
+ * GetThreadPriority:
+ * @param handle: The thread handle to query.
+ *
+ * Gets the priority of the given thread.
+ * @return: A MonoThreadPriority approximating the current POSIX 
+ * thread priority, or THREAD_PRIORITY_NORMAL on error.
+ */
+gint32 GetThreadPriority (gpointer handle)
+{
+	struct _WapiHandle_thread *thread_handle;
+	int policy;
+	struct sched_param param;
+	gboolean ok = _wapi_lookup_handle (handle, WAPI_HANDLE_THREAD,
+				  (gpointer *)&thread_handle);
+				  
+	if (ok == FALSE) {
+/*
+		g_warning ("%s: error looking up thread handle %p", __func__,
+			   handle);
+*/
+		
+		return (THREAD_PRIORITY_NORMAL);
+	}
+	
+	switch (pthread_getschedparam (thread_handle->id, &policy, &param)) {
+		case 0:
+			return (_wapi_thread_posix_priority_to_priority (param.sched_priority, policy));
+		case ESRCH:
+			g_warning ("pthread_getschedparam: error looking up thread id %x", (gsize)thread_handle->id);
+	}
+	
+	return (THREAD_PRIORITY_NORMAL);
+}
+
+/**
+ * SetThreadPriority:
+ * @param handle: The thread handle to query.
+ * @param priority: The priority to give to the thread.
+ *
+ * Sets the priority of the given thread.
+ * @return: TRUE on success, FALSE on failure or error.
+ */
+gboolean SetThreadPriority (gpointer handle, gint32 priority)
+{
+	struct _WapiHandle_thread *thread_handle;
+	int policy,
+	    posix_priority,
+	    rv;
+	struct sched_param param;
+	gboolean ok = _wapi_lookup_handle (handle, WAPI_HANDLE_THREAD,
+				  (gpointer *)&thread_handle);
+				  
+	if (ok == FALSE) {
+/*
+		g_warning ("%s: error looking up thread handle %p", __func__,
+			   handle);
+*/
+		
+		return ok;
+	}
+	
+	rv = pthread_getschedparam (thread_handle->id, &policy, &param);
+	if (rv) {
+		if (ESRCH == rv)
+			g_warning ("pthread_getschedparam: error looking up thread id %x", (gsize)thread_handle->id);
+		return FALSE;
+	}
+	
+	posix_priority =  _wapi_thread_priority_to_posix_priority (priority, policy);
+	if (0 > posix_priority)
+		return FALSE;
+		
+	param.sched_priority = posix_priority;
+	switch (pthread_setschedparam (thread_handle->id, policy, &param)) {
+		case 0:
+			return TRUE;
+		case ESRCH:
+			g_warning ("pthread_setschedparam: error looking up thread id %x", (gsize)thread_handle->id);
+			break;
+		case ENOTSUP:
+			g_warning ("%s: priority %d not supported", __func__, priority);
+			break;
+		case EPERM:
+			g_warning ("%s: permission denied", __func__);
+			break;
+	}
+	
+	return FALSE;
+}
+
diff --git a/mono/metadata/icall-def.h b/mono/metadata/icall-def.h
index 63fb708..bb5ecdf 100644
--- a/mono/metadata/icall-def.h
+++ b/mono/metadata/icall-def.h
@@ -871,6 +871,7 @@ ICALL(THREAD_4, "FreeLocalSlotValues", mono_thread_free_local_slot_values)
 ICALL(THREAD_55, "GetAbortExceptionState", ves_icall_System_Threading_Thread_GetAbortExceptionState)
 ICALL(THREAD_7, "GetDomainID", ves_icall_System_Threading_Thread_GetDomainID)
 ICALL(THREAD_8, "GetName_internal(System.Threading.InternalThread)", ves_icall_System_Threading_Thread_GetName_internal)
+ICALL(THREAD_56, "GetPriority_internal(System.Threading.InternalThread)", ves_icall_System_Threading_Thread_GetPriority_internal)
 ICALL(THREAD_11, "GetState(System.Threading.InternalThread)", ves_icall_System_Threading_Thread_GetState)
 ICALL(THREAD_53, "Interrupt_internal(System.Threading.InternalThread)", ves_icall_System_Threading_Thread_Interrupt_internal)
 ICALL(THREAD_12, "Join_internal(System.Threading.InternalThread,int,intptr)", ves_icall_System_Threading_Thread_Join_internal)
@@ -878,6 +879,7 @@ ICALL(THREAD_13, "MemoryBarrier", ves_icall_System_Threading_Thread_MemoryBarrie
 ICALL(THREAD_14, "ResetAbort_internal()", ves_icall_System_Threading_Thread_ResetAbort)
 ICALL(THREAD_15, "Resume_internal()", ves_icall_System_Threading_Thread_Resume)
 ICALL(THREAD_18, "SetName_internal(System.Threading.InternalThread,string)", ves_icall_System_Threading_Thread_SetName_internal)
+ICALL(THREAD_57, "SetPriority_internal(System.Threading.InternalThread,System.Threading.ThreadPriority)", ves_icall_System_Threading_Thread_SetPriority_internal)
 ICALL(THREAD_21, "SetState(System.Threading.InternalThread,System.Threading.ThreadState)", ves_icall_System_Threading_Thread_SetState)
 ICALL(THREAD_22, "Sleep_internal", ves_icall_System_Threading_Thread_Sleep_internal)
 ICALL(THREAD_54, "SpinWait_nop", ves_icall_System_Threading_Thread_SpinWait_nop)
diff --git a/mono/metadata/threads-types.h b/mono/metadata/threads-types.h
index b090fcb..edb6bcf 100644
--- a/mono/metadata/threads-types.h
+++ b/mono/metadata/threads-types.h
@@ -67,6 +67,8 @@ gint32 ves_icall_System_Threading_Thread_GetDomainID (void) MONO_INTERNAL;
 gboolean ves_icall_System_Threading_Thread_Yield (void) MONO_INTERNAL;
 MonoString* ves_icall_System_Threading_Thread_GetName_internal (MonoInternalThread *this_obj) MONO_INTERNAL;
 void ves_icall_System_Threading_Thread_SetName_internal (MonoInternalThread *this_obj, MonoString *name) MONO_INTERNAL;
+gint32 ves_icall_System_Threading_Thread_GetPriority_internal (MonoInternalThread *this_obj) MONO_INTERNAL;
+void ves_icall_System_Threading_Thread_SetPriority_internal (MonoInternalThread *this_obj, gint32 priority) MONO_INTERNAL;
 MonoObject* ves_icall_System_Threading_Thread_GetCachedCurrentCulture (MonoInternalThread *this_obj) MONO_INTERNAL;
 void ves_icall_System_Threading_Thread_SetCachedCurrentCulture (MonoThread *this_obj, MonoObject *culture) MONO_INTERNAL;
 MonoObject* ves_icall_System_Threading_Thread_GetCachedCurrentUICulture (MonoInternalThread *this_obj) MONO_INTERNAL;
diff --git a/mono/metadata/threads.c b/mono/metadata/threads.c
index 1b421c4..756d2c1 100755
--- a/mono/metadata/threads.c
+++ b/mono/metadata/threads.c
@@ -1342,6 +1342,43 @@ ves_icall_System_Threading_Thread_SetName_internal (MonoInternalThread *this_obj
 	mono_thread_set_name_internal (this_obj, name, TRUE);
 }
 
+/*
+ * ves_icall_System_Threading_Thread_GetPriority_internal:
+ * @param this_obj: The MonoInternalThread on which to operate.
+ *
+ * Gets the priority of the given thread.
+ * @return: The priority of the given thread.
+ */
+gint32
+ves_icall_System_Threading_Thread_GetPriority_internal (MonoInternalThread *this_obj)
+{
+  gint32 priority;
+ 
+  ensure_synch_cs_set (this_obj);
+  EnterCriticalSection (this_obj->synch_cs);
+  
+  priority = GetThreadPriority (this_obj->handle) + 2;
+  
+  LeaveCriticalSection (this_obj->synch_cs);
+  return priority;
+}
+ 
+/* 
+ * ves_icall_System_Threading_Thread_SetPriority_internal:
+ * @param this_obj: The MonoInternalThread on which to operate.
+ * @param priority: The priority to set.
+ *
+ * Sets the priority of the given thread.
+ */
+void 
+ves_icall_System_Threading_Thread_SetPriority_internal (MonoInternalThread *this_obj, gint32 priority)
+{
+  ensure_synch_cs_set (this_obj);
+  EnterCriticalSection (this_obj->synch_cs);
+  SetThreadPriority (this_obj->handle, priority - 2);
+  LeaveCriticalSection (this_obj->synch_cs);
+}
+
 /* If the array is already in the requested domain, we just return it,
    otherwise we return a copy in that domain. */
 static MonoArray*
-- 
1.9.1

Just for clarification: Running the mono application using 'chrt --rr 10' makes all new threads use the SCHED_RR policy, which supports priorities from 1 to 99 (sched_get_priority_min(SCHED_RR) to sched_get_priority_max(SCHED_RR) ).
New threads created with pthread_create calls with attr == NULL, is created with default attributes, and the default value of the inherit scheduler attribute happens to be PTHREAD_INHERIT_SHED. This means that the chrt -rr approach also works for all sub-threads and not just the "outer thread", since new threads in mono are created with default attributes as far as I can tell (the create thread code is pretty wrapped up in macros - so I'm not 100% sure of this).

There are as I see it three different drawbacks to this solutions:

1. We need to run the application with chrt --rr 10
2. We need to run the application as root.
3. Running all threads with policy SCHED_RR makes it impossible to get a lower priority than 1, which is higher than that of both SCHED_NORMAL and SCHED_IDLE, meaning that all mono threads will have a higher priority than normal threads on the system, even with ThreadPriority = Lowest.

I have a suggestion that might get rid of (1) and (3) but not (2).
Experiments with various combinations of policy and priority I have come to the conclusion that simply mapping ThreadPriority values to pthread priorities is not enough. But if the policy is also changed better results can be achieved.
Consider the following mapping:
ThreadPriority.Lowest : SCHED_IDLE
ThreadPriority.BelowNormal : ?????
ThreadPriority.Normal : SCHED_NORMAL
ThreadPriority.AboveNormal : SCHED_RR with sched_priority=ched_get_priority_min(SCHED_RR)
ThreadPriority.Highest : SCHED_RR with sched_priority=ched_get_priority_max(SCHED_RR)

I don't yet know what to do with the ThreadPriority.BelowNormal priority. Ideas are welcome.

I meant to write a whole bunch about this when I originally updates this patch but other events unfortunately got in the way and now my memory is failing and I can't find my notes (if I wrote any).

However, AFAIR the problem does indeed boil down to the fact that only root can set SCHED_RR, as it has the potential to lock out all the SCHED_NORMAL processes. You can also use chrt to change the schedule of an existing non-root process, though until very recently Mono made it pretty much impossible to relate a .NET thread to a system thread (on purpose since there's no guarantee they are 1:1). However, I believe recent versions of Mono have surfaced the .NET thread name to the OS thread name, though I haven't tested this out myself.

Needless to say, this is super messy and not going to be used much as most people will not want to risk a SCHED_RR capable Mono process locking out the rest of the system. Perhaps Mono would accept a patch that tried to set it anyway and simply did nothing if mono wasn't running as root, though I kind of doubt it.

I agree it would be easier to handle BelowNormal and Lowest, maybe with SCHED_IDLE as you suggest or simply by nicing them with different priorities.

However, my own need was for AboveNormal/Highest and has gone away for now (though it may well return next year). I was experimenting with this because I work with a project (http://opensimulator.org) that has some time critical scene/packet-handling operations that might have been helped by increasing thread priority. But initial tests were not encouraging and I ended up improving performance in other ways instead. That's also why I didn't further fix some of the other issues of the patch after hacking it out of the Bugzilla report.

But I'd be very happy to keep discussing this if you want (I'm curious what you need it for), though I might not be able to help you much further. Although as I said, I may end up looking at it again next year (assuming you haven't fixed it and got a mono patch in ;).

A have completely rewritten the patch to use "nice" instead of pthread_setschedparam through the posix calls setpriority(2).
It even works for non-root users as long as the priority is only lowered.
I posted it here: https://gist.github.com/aasimon/c8ae6fc3cf5d9b82b6ca