SObject switch and polymorphic SObject fields

How I expected it to work.

List<User> notifyUsers = new List<User>();
List<Group> notifyGroups = new List<Group>();

List<Property__c> properties = [SELECT 
                                    TYPEOF Owner 
                                        WHEN User THEN Id, Name, Username, Email 
                                        WHEN Group THEN Id, Name, Email, DoesSendEmailToMembers 
                                    END 
                                FROM Property__c
                                WHERE ID in Trigger.new];

for (Property__c prop: properties) {
    switch on prop.Owner {
        when User u {
            notifyUsers.add(u);
        }
        when Group g {
            notifyGroups.add(g);
        }
    }
}

How it actually works

List<User> notifyUsers = new List<User>();
List<Group> notifyGroups = new List<Group>();

List<Property__c> properties = [SELECT 
                                    TYPEOF Owner 
                                        WHEN User THEN Id, Name, Username, Email 
                                        WHEN Group THEN Id, Name, Email, DoesSendEmailToMembers 
                                    END 
                                FROM Property__c
                                WHERE ID in Trigger.new];

for (Property__c prop: properties) {
    switch on (SObject) prop.Owner {
        when User u {
            notifyUsers.add(u);
        }
        when Group g {
            notifyGroups.add(g);
        }
    }
}