Topmost window via CGWindow

gistfile1.m

NSWindow *topmostAppWindowAtPoint(CGPoint screenPoint)
{
    const CGWindowLevel kScreensaverWindowLevel = CGWindowLevelForKey(kCGScreenSaverWindowLevelKey);
    
    /*  This function returns a pointer to the app's topmost NSWindow that
        the point `screenPoint` is over. The important distinction here is that
        this function takes _all_ system windows into consideration and will return
        `nil` if there is a system window (or NSMenu etc.) that the cursor
        is over which is atop the app window, which is information that
        can't otherwise be gleaned by checking against `[NSApp orderedWindows]` etc.
        
        This is useful in cases where you need to know e.g. what window the cursor is
        *really* over, whether it is a window in your app, or a menu in the menubar, etc.
     */
    
    pid_t appPID = [[NSRunningApplication currentApplication] processIdentifier];
    
    NSWindow *appWindow = nil;
    CFArrayRef windowArray = CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, kCGNullWindowID);
    CFIndex windowCount = 0;
    
    NSRect mainScreenRect = NSScreen.mainScreen.frame;
    float mainScreenHeight = mainScreenRect.size.height;
    
    /* Flip the screen coord point Y here to match what the CGWindow expects */
    screenPoint.y = mainScreenHeight - screenPoint.y;
    
    if ((windowCount = CFArrayGetCount(windowArray)))
    {
        for (CFIndex i = 0; i < windowCount; i++)
        {
            NSDictionary *windowInfoDictionary = (__bridge NSDictionary *)((CFDictionaryRef)CFArrayGetValueAtIndex(windowArray, i));
            NSNumber *ownerPID = (NSNumber *)(windowInfoDictionary[(id)kCGWindowOwnerPID]);
            NSNumber *level = (NSNumber *)(windowInfoDictionary[(id)kCGWindowLayer]);
            
            if (ownerPID.intValue == appPID)
            {
                /*  We are concerned with menus or application windows etc., anything
                 at or above screensaver level (1000) we just assume is something special
                 that we should ignore, and skip it. */
                 
                if (level.integerValue < kScreensaverWindowLevel)
                {
                    CGRect rect = NSZeroRect;
                    CGRectMakeWithDictionaryRepresentation((CFDictionaryRef)(windowInfoDictionary[(id)kCGWindowBounds]), &rect);
                    
                    if (CGRectContainsPoint(rect, screenPoint))
                    {
                        NSNumber *windowID = windowInfoDictionary[(id)kCGWindowNumber];
                        appWindow = [NSApp windowWithWindowNumber:windowID.integerValue];
                        
                        break;
                    }
                }
            }
        }
    }
        
    CFRelease(windowArray);
    
    return appWindow;
}

This only works for windows owned by this code executing app https://developer.apple.com/documentation/appkit/nsapplication/1428464-window

Please read the code comment carefully:

 /*  This function returns a pointer to the app's topmost NSWindow that
        the point `screenPoint` is over. The important distinction here is that
        this function takes _all_ system windows into consideration and will return
        `nil` if there is a system window (or NSMenu etc.) that the cursor
        is over which is atop the app window, which is information that
        can't otherwise be gleaned by checking against `[NSApp orderedWindows]` etc.
        
        This is useful in cases where you need to know e.g. what window the cursor is
        *really* over, whether it is a window in your app, or a menu in the menubar, etc.
     */

My bad. Thanks.