TokenService.cs

    public class TokenService
    {
        public static IEnumerable<Claim> ValidateAndParseToClaims(string token, string validAudience)
        {
            var parameters = CreateTokenValidationParameters(validAudience);

            SecurityToken jwt;
            var principal = new JwtSecurityTokenHandler().ValidateToken(token, parameters, out jwt);
            
            return principal.Claims;
        }

        // Requires Nuget-package System.IdentityModel.Tokens.Jwt
        private static TokenValidationParameters CreateTokenValidationParameters(string validAudience)
        {
            //Can be retrieved dynamically x5c from /core/.well-known/jwks
            var rawData = Encoding.UTF8.GetBytes("MIIDBTCCAfGgAwIBAgIQNQb+T2ncIrNA6cKvUA1GWTAJBgUrDgMCHQUAMBIxEDAOBgNVBAMTB0RldlJvb3QwHhcNMTAwMTIwMjIwMDAwWhcNMjAwMTIwMjIwMDAwWjAVMRMwEQYDVQQDEwppZHNydjN0ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqnTksBdxOiOlsmRNd+mMS2M3o1IDpK4uAr0T4/YqO3zYHAGAWTwsq4ms+NWynqY5HaB4EThNxuq2GWC5JKpO1YirOrwS97B5x9LJyHXPsdJcSikEI9BxOkl6WLQ0UzPxHdYTLpR4/O+0ILAlXw8NU4+jB4AP8Sn9YGYJ5w0fLw5YmWioXeWvocz1wHrZdJPxS8XnqHXwMUozVzQj+x6daOv5FmrHU1r9/bbp0a1GLv4BbTtSh4kMyz1hXylho0EvPg5p9YIKStbNAW9eNWvv5R8HN7PPei21AsUqxekK0oW9jnEdHewckToX7x5zULWKwwZIksll0XnVczVgy7fCFwIDAQABo1wwWjATBgNVHSUEDDAKBggrBgEFBQcDATBDBgNVHQEEPDA6gBDSFgDaV+Q2d2191r6A38tBoRQwEjEQMA4GA1UEAxMHRGV2Um9vdIIQLFk7exPNg41NRNaeNu0I9jAJBgUrDgMCHQUAA4IBAQBUnMSZxY5xosMEW6Mz4WEAjNoNv2QvqNmk23RMZGMgr516ROeWS5D3RlTNyU8FkstNCC4maDM3E0Bi4bbzW3AwrpbluqtcyMN3Pivqdxx+zKWKiORJqqLIvN8CT1fVPxxXb/e9GOdaR8eXSmB0PgNUhM4IjgNkwBbvWC9F/lzvwjlQgciR7d4GfXPYsE1vf8tmdQaY8/PtdAkExmbrb9MihdggSoGXlELrPA91Yce+fiRcKY3rQlNWVd4DOoJ/cPXsXwry8pWjNCo5JD8Q+RQ5yZEy7YPoifwemLhTdsBz3hlZr28oCGJ3kbnpW0xGvQb3VHSTVVbeei0CfXoW6iz1");
            var x509Certificate2 = new X509Certificate2(rawData);
            var x509SecurityToken = new X509SecurityToken(x509Certificate2);
            var parameters = new TokenValidationParameters
            {
                ValidAudience = validAudience,
                ValidIssuer = "https://idsrv3.com",
                IssuerSigningToken = x509SecurityToken
            };
            return parameters;
        }
    }

Thank you Loctanvo, I've at least got my ValidateToken working from your code using a Flow of ResourceOwner and AccessTokenType.Jwt which was my first goal.

Next I'm going to have to try and figure out a way to create a claimsPrincipal from a reference token. (I.e. Service to service calls)

FYI. For anybody else, I couldn't use ClientId as the validAudience. I had to change it to https://localhost/idsvr/resources as that's what came in as the audience

Glad I could help, @kkfrosty . Forgot that you're trying to validate access tokens and not id tokens. For access tokens, https://localhost/idsvr/resources is the audience. For id tokens, it should be the client id.