TaskCompletionSourceDictionary - key is generic, value can be any result, with runtime type check to ensure producer and consumer type parity

TaskCompletionSourceDictionary.cs

[DebuggerDisplay("Count = {" + nameof(Count) + "}")]
public sealed class TaskCompletionSourceDictionary
{
    private readonly TaskCompletionSourceDictionary<Type> _dictionary = new TaskCompletionSourceDictionary<Type>();

    public int Count => _dictionary.Count;

    public bool TryGetValue<T>(out TaskCompletionSource<T> taskCompletionSource) =>
        _dictionary.TryGetValue(typeof(T), out taskCompletionSource);

    public TaskCompletionSource<T> GetOrAdd<T>() =>
        _dictionary.GetOrAdd<T>(typeof(T));

    public bool TryAdd<T>() => _dictionary.TryAdd(typeof(T));

    public bool TryRemove<T>(out TaskCompletionSource<T> taskCompletionSource) =>
        _dictionary.TryRemove(typeof(T), out taskCompletionSource);
}

TaskCompletionSourceDictionary`1.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;

[DebuggerDisplay("Count = {" + nameof(Count) + "}")]
public sealed class TaskCompletionSourceDictionary<TKey>
{
    private readonly ConcurrentDictionary<TKey, TaskCompletionSource> _dictionary;

    public TaskCompletionSourceDictionary() =>
        _dictionary = new ConcurrentDictionary<TKey, TaskCompletionSource>();

    public TaskCompletionSourceDictionary(IEqualityComparer<TKey> equalityComparer) =>
        _dictionary = new ConcurrentDictionary<TKey, TaskCompletionSource>(equalityComparer);

    public int Count => _dictionary.Count;

    public bool TryGetValue<TValue>(TKey key, out TaskCompletionSource<TValue> taskCompletionSource)
    {
        var result = _dictionary.TryGetValue(key, out var tcs);
        taskCompletionSource = result ? tcs.Cast<TValue>() : default;
        return result;
    }

    public TaskCompletionSource<TValue> GetOrAdd<TValue>(TKey key)
    {
        var taskCompletionSource = _dictionary.GetOrAdd(key, Factory);
        return taskCompletionSource.Cast<TValue>();
    }

    public bool TryAdd(TKey key)
    {
        _dictionary.GetOrAdd(key, Factory);
        return true;
    }

    public bool TryRemove<TValue>(TKey key, out TaskCompletionSource<TValue> taskCompletionSource)
    {
        var result = _dictionary.TryRemove(key, out var tcs);
        taskCompletionSource = result ? tcs.Cast<TValue>() : default;
        return result;
    }

    private static TaskCompletionSource Factory(TKey _) => TaskCompletionSource.Create<TKey>();

    private sealed class TaskCompletionSource
    {
        private readonly object _taskCompletionSource;
        private readonly Type _type;

        private TaskCompletionSource(Type type, object taskCompletionSource)
        {
            _type = type;
            _taskCompletionSource = taskCompletionSource;
        }

        public static TaskCompletionSource Create<T>() =>
            new TaskCompletionSource(
                typeof(T),
                new TaskCompletionSource<T>());

        public TaskCompletionSource<T> Cast<T>()
        {
            if (typeof(T) != _type)
                throw new InvalidCastException($"Invalid cast from '{_type}' to '{typeof(T)}'.");

            return (TaskCompletionSource<T>) _taskCompletionSource;
        }
    }
}