RudeWindowManagerExt.c

#include <windows.h>
#include <dwmapi.h>
#include <tlhelp32.h>

// Callback function that marks a window as rude or non-rude depending on the parameter passed to MarkRudeWindow(HWND hWnd, BOOL bRude)
BOOL EnumWindowsProc(HWND hWnd, LPARAM lParam)
{
    static UINT uShellHookMessage = 0;

    WCHAR rgwcClassName[256] = {0};
    HWND *rghWnd = (HWND *)lParam;

    if (!uShellHookMessage)
        uShellHookMessage = RegisterWindowMessageW(L"SHELLHOOK");
    GetClassNameW(hWnd, rgwcClassName, 256);
    if (!wcscmp(L"WorkerW", rgwcClassName) || !wcscmp(L"WorkerA", rgwcClassName))
        PostMessageW(hWnd, uShellHookMessage, (WPARAM){rghWnd[0] ? 0x35 : 0x36}, (LPARAM)rghWnd[1]);

    return TRUE;
}

// Marks a window as rude or non-rude.
VOID MarkRudeWindow(HWND hWnd, BOOL bRude)
{
    // https://learn.microsoft.com/en-us/windows/win32/api/shobjidl_core/nf-shobjidl_core-itaskbarlist2-markfullscreenwindow#remarks
    if (bRude)
        RemovePropW(hWnd, L"NonRudeHWND");
    else
        SetPropW(hWnd, L"NonRudeHWND", (HANDLE)TRUE);
    EnumWindows(EnumWindowsProc, (LPARAM)((HWND[]){(HWND)((LONG_PTR)bRude), hWnd}));
}

/*
Determine the rudeness of a window.
A window is considered rude by this function if:
1. It is not anyway visible on the user's desktop.
2. It is not a fullscreen window.
3. It is not minimized.
4. It is not cloaked.
5. It has the WS_VISIBLE window style.
6. Lacks the WS_EX_NOACTIVATE and WS_EX_TRANSPARENT extended window styles.

The Windows Shell will consider any window rude if:
1. It is not cloaked.
2. It has the VS_VISIBLE window style.
3. A window obscures the entire desktop it is present on.
*/
BOOL IsWindowRude(HWND hWnd)
{
    WINDOWINFO wi = {.cbSize = sizeof(WINDOWINFO)};
    MONITORINFO mi = {.cbSize = sizeof(MONITORINFO)};
    LONG lWidth = 0, lHeight = 0;
    BOOL bCloaked = FALSE;
    BYTE bAlpha = 255;
    DWORD dwExtendedWindowStyles = GetWindowLongPtrW(hWnd, GWL_EXSTYLE);

    DwmGetWindowAttribute(hWnd, DWMWA_CLOAKED, &bCloaked, sizeof(BOOL));
    GetLayeredWindowAttributes(hWnd, NULL, &bAlpha, &((DWORD){LWA_ALPHA}));
    GetWindowInfo(hWnd, &wi);
    GetMonitorInfoW(MonitorFromWindow(hWnd, MONITOR_DEFAULTTONEAREST), &mi);

    return IsWindowVisible(hWnd) &&
                   !IsIconic(hWnd) &&
                   !bCloaked &&
                   bAlpha &&
                   !(dwExtendedWindowStyles & WS_EX_NOACTIVATE) &&
                   !(dwExtendedWindowStyles & WS_EX_TRANSPARENT) &&
                   (lWidth = wi.rcWindow.right - wi.rcWindow.left) ==
                       wi.rcClient.right - wi.rcClient.left &&
                   (lHeight = wi.rcWindow.bottom - wi.rcWindow.top) ==
                       wi.rcClient.bottom - wi.rcClient.top
               ? lWidth == mi.rcMonitor.right - mi.rcMonitor.left &&
                     lHeight == mi.rcMonitor.bottom - mi.rcMonitor.top &&
                     wi.rcWindow.left == mi.rcMonitor.left &&
                     wi.rcWindow.top == mi.rcMonitor.top
               : FALSE;
}

void WinEventProc(
    HWINEVENTHOOK hWinEventHook,
    DWORD event,
    HWND hwnd,
    LONG idObject,
    LONG idChild,
    DWORD idEventThread,
    DWORD dwmsEventTime)
{
    if (idObject && idChild)
        return;
    WCHAR rgwcClassName[256] = {0};
    GetClassNameW(hwnd, rgwcClassName, 256);
    if (wcslen(rgwcClassName) && hwnd != GetShellWindow())
        MarkRudeWindow(hwnd, IsWindowRude(hwnd));
}

// Callback function when recalculating the rude state of windows on the input desktop.
BOOL EnumThreadWindowsProc(HWND hWnd, LPARAM lParam)
{
    WinEventProc(NULL, 0, hWnd, OBJID_WINDOW, CHILDID_SELF, 0, 0);
    return TRUE;
}

LRESULT WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
    static UINT uTaskbarCreatedMessage = 0;
    if (uMsg == WM_CREATE && !uTaskbarCreatedMessage)
    {
        uTaskbarCreatedMessage = RegisterWindowMessageW(L"TaskbarCreated");
        PostMessageW(hWnd, uTaskbarCreatedMessage, 0, 0);
    }
    // Enumerate through all threads on the system, reason being the EnumWindowsProc function actually doesn't enumerate UWP windows.
    else if (uMsg == uTaskbarCreatedMessage)
    {
        HANDLE hSnapshot = NULL;
        THREADENTRY32 te = {.dwSize = sizeof(THREADENTRY32)};
        if (Thread32First((hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0)), &te))
            do
                EnumThreadWindows(te.th32ThreadID, EnumThreadWindowsProc, 0);
            while (Thread32Next(hSnapshot, &te));
        CloseHandle(hSnapshot);
    }
    return DefWindowProcW(hWnd, uMsg, wParam, lParam);
}

INT wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, INT nShowCmd)
{
    MSG msg = {0};
    HANDLE hThread = GetCurrentThread(), hProcess = GetCurrentProcess();

    // This prevent the operating system from giving the process any form of priority.
    SetPriorityClass(hProcess, PROCESS_MODE_BACKGROUND_BEGIN);
    SetThreadPriority(hThread, THREAD_MODE_BACKGROUND_BEGIN);
    SetThreadPriorityBoost(hThread, TRUE);
    SetProcessPriorityBoost(hProcess, TRUE);
    CloseHandle(hProcess);
    CloseHandle(hThread);

    // Register a window class to recalculate the rude state incase the Windows Shell is relaunched.
    RegisterClassExW(&((WNDCLASSEXW){.cbSize = sizeof(WNDCLASSEXW), .lpszClassName = L"RudeWindowManagerExt", .lpfnWndProc = WndProc, .hInstance = hInstance}));
    CreateWindowExW(0, L"RudeWindowManagerExt", NULL, 0, 0, 0, 0, 0, HWND_MESSAGE, NULL, hInstance, NULL);

    // Setup event hooks to observe window foreground status and movement.
    SetWinEventHook(EVENT_SYSTEM_FOREGROUND, EVENT_SYSTEM_FOREGROUND, NULL, WinEventProc, 0, 0, WINEVENT_OUTOFCONTEXT);
    SetWinEventHook(EVENT_OBJECT_LOCATIONCHANGE, EVENT_OBJECT_LOCATIONCHANGE, NULL, WinEventProc, 0, 0, WINEVENT_OUTOFCONTEXT);

    while (GetMessageW(&msg, NULL, 0, 0))
    {
        TranslateMessage(&msg);
        DispatchMessageW(&msg);
    }
    return 0;
}