A modified version of SerializedDictionary (https://github.com/azixMcAze/Unity-SerializableDictionary) that has one KeyValuePair for each given enum value

EnumMap.cs

using System;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;

namespace DerHugo
{
#if UNITY_EDITOR
    /// <summary>
    /// Non generic base class for <see cref="EnumMap{TKey,TValue}"/> for drawers
    /// </summary>
    public abstract class EnumMap
    {
        public abstract string GetPairsName { get; }
    }
#endif

    /// <summary>
    /// Wrapper class for mapping enums to specific value
    /// </summary>
    /// <typeparam name="TKey"></typeparam>
    /// <typeparam name="TValue"></typeparam>
    [Serializable]
    public class EnumMap<TKey, TValue>
#if UNITY_EDITOR
        : EnumMap, ISerializationCallbackReceiver
#endif
        where TKey : Enum
    {
        [SerializeField]
        private List<EnumValuePair<TKey, TValue>> m_KeyValuePairs;

        private Dictionary<TKey, TValue> m_Dictionary;

        public TValue this[TKey key]
        {
            get => m_Dictionary[key];
            set => m_Dictionary[key] = value;
        }
        
#if UNITY_EDITOR
        public override string GetPairsName => nameof(m_KeyValuePairs);
        
        public void OnBeforeSerialize()
        {
            // not needed for now
        }

        public void OnAfterDeserialize()
        {
            m_Dictionary = new Dictionary<TKey, TValue>(m_KeyValuePairs.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
        }
#endif

        public EnumMap()
        {
            var enums = (TKey[])Enum.GetValues(typeof(TKey));

            m_KeyValuePairs = enums.Select(item => new EnumValuePair<TKey, TValue>(item)).ToList();
            m_Dictionary = new Dictionary<TKey, TValue>(m_KeyValuePairs.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
        }

        public EnumMap(IEnumerable<TKey> keys)
        {
            m_KeyValuePairs = keys.Select(item => new EnumValuePair<TKey, TValue>(item)).ToList();
            m_Dictionary = new Dictionary<TKey, TValue>(m_KeyValuePairs.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
        }
        
        public EnumMap(IEnumerable<EnumValuePair<TKey, TValue>> pairs)
        {
            m_KeyValuePairs = pairs.ToList();
            m_Dictionary = new Dictionary<TKey, TValue>(m_KeyValuePairs.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
        }
    }
}

EnumMapDrawer.cs

#if UNITY_EDITOR
using System;
using UnityEditor;
using UnityEngine;

namespace DerHugo.Editor
{
    [CustomPropertyDrawer(typeof(EnumMap), true)]
    public class EnumMapDrawer : PropertyDrawer
    {
        public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
        {
            var height = EditorGUIUtility.singleLineHeight;

            if (property.isExpanded)
            {
                var target = property.GetTargetObjectOfProperty<EnumMap>();
                var pairs = property.FindPropertyRelative(target.GetPairsName);

                for (var i = 0; i < pairs.arraySize; i++)
                {
                    var element = pairs.GetArrayElementAtIndex(i);

                    height += EditorGUI.GetPropertyHeight(element, true);
                }
            }

            return height;
        }

        public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
        {
            EditorGUI.BeginProperty(position, label, property);

            var lineHeight = EditorGUIUtility.singleLineHeight;

            var foldoutRect = position;
            foldoutRect.height = lineHeight;
            property.isExpanded = EditorGUI.Foldout(foldoutRect, property.isExpanded, label);

            position.y += lineHeight;
            position.height -= lineHeight;

            if (property.isExpanded)
            {
                EditorGUI.indentLevel++;
                position = EditorGUI.IndentedRect(position);
                position.height = lineHeight;
                EditorGUI.indentLevel--;

                var target = property.GetTargetObjectOfProperty<EnumMap>();
                var pairs = property.FindPropertyRelative(target.GetPairsName);

                for (var i = 0; i < pairs.arraySize; i++)
                {
                    var element = pairs.GetArrayElementAtIndex(i);

                    var elementHeight = EditorGUI.GetPropertyHeight(element, true);
                    position.height = elementHeight;
                    EditorGUI.PropertyField(position, element, GUIContent.none, true);

                    position.y += elementHeight;
                }
            }

            EditorGUI.EndProperty();
        }
    }
}
#endif

EnumValuePair.cs

using System;

namespace DerHugo
{
    [Serializable]
    public class EnumValuePair<TKey, TValue> : EnumValuePair where TKey : Enum
    {
        public TKey Key;
        public TValue Value;

        public EnumValuePair() { }

        public EnumValuePair(TKey key)
        {
            Key = key;
        }

        public EnumValuePair(TKey key, TValue value)
        {
            Key = key;
            Value = value;
        }

#if UNITY_EDITOR
        public override string GetKeyName => nameof(Key);
        public override string GetValueName => nameof(Value);
#endif
    }

    [Serializable]
    public abstract class EnumValuePair
    {
#if UNITY_EDITOR
        public abstract string GetKeyName { get; }
        public abstract string GetValueName { get; }
#endif
    }
}

EnumValuePairDrawer.cs

#if UNITY_EDITOR
using System;
using UnityEditor;
using UnityEngine;

namespace DerHugo.Editor
{
    [CustomPropertyDrawer(typeof(EnumValuePair), true)]
    public class EnumValuePairDrawer : PropertyDrawer
    {
        private const float spacing = 10;
    
        public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
        {
            var target = property.GetTargetObjectOfProperty<EnumValuePair>();
            var value = property.FindPropertyRelative(target.GetValueName);

            return EditorGUI.GetPropertyHeight(value, true);
        }

        public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
        {
            var target = property.GetTargetObjectOfProperty<EnumValuePair>();
            var key = property.FindPropertyRelative(target.GetKeyName);
            var value = property.FindPropertyRelative(target.GetValueName);
        
            var expanded = value.isExpanded;
            value.isExpanded = true;
            var height = EditorGUI.GetPropertyHeight(value, true);
            value.isExpanded = expanded;
            var isMultiLevel = height > EditorGUIUtility.singleLineHeight;
        
            EditorGUI.BeginProperty(position, label, property);

            using (new EditorGUI.DisabledScope(true))
            {
                var labelRect = position;
                labelRect.height = EditorGUIUtility.singleLineHeight;
                labelRect.width = EditorGUIUtility.labelWidth - spacing;

                if (isMultiLevel)
                {
                    value.isExpanded = EditorGUI.Foldout(labelRect, value.isExpanded, key.enumDisplayNames[key.enumValueIndex], true, EditorStyles.popup);
                }
                else
                {
                    EditorGUI.PropertyField(labelRect, key, GUIContent.none);
                }
            }

            position.x += EditorGUIUtility.labelWidth + spacing;
            position.width -= EditorGUIUtility.labelWidth + spacing;
        
            if (isMultiLevel)
            {
                EditorGUI.PropertyField(position, value, GUIContent.none, true);
            }
            else
            {
                EditorGUI.PropertyField(position, value, GUIContent.none);
            }

            EditorGUI.EndProperty();
        }
    }
}
#endif

SerializedPropertyExtensions.cs

#if UNITY_EDITOR
using System;
using System.Collections;
using System.Reflection;
using UnityEditor;
using UnityEngine;

namespace DerHugo.Editor
{
    public static class SerializedPropertyExtensions
    {
        private const char k_PropertyPathSeparator = '.';
        private const string k_ArrayData = ".Array.data[";
        private const string k_ArrayBegin = "[";
        private const string k_ArrayEnd = "]";

        /// <summary>
        /// Gets the object the property represents.
        /// </summary>
        public static T GetTargetObjectOfProperty<T>(this SerializedProperty prop)
        {
            if (prop == null)
            {
                return default;
            }

            var path = prop.propertyPath.Replace(k_ArrayData, k_ArrayBegin);
            object obj = prop.serializedObject.targetObject;
            var elements = path.Split(k_PropertyPathSeparator);
            foreach (var element in elements)
            {
                if (element.Contains(k_ArrayBegin))
                {
                    var i = element.IndexOf(k_ArrayBegin);
                    var elementName = element.Substring(0, i);
                    var index = Convert.ToInt32(element.Substring(i).Replace(k_ArrayBegin, "").Replace(k_ArrayEnd, ""));
                    obj = GetValue_Imp(obj, elementName, index);
                }
                else
                {
                    obj = GetValue_Imp(obj, element);
                }
            }

            return (T)obj;
        }

        private static object GetValue_Imp(object source, string name)
        {
            if (source == null)
            {
                return null;
            }

            var type = source.GetType();

            while (type != null)
            {
                var f = type.GetField(name, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance);

                if (f != null)
                {
                    return f.GetValue(source);
                }

                var p = type.GetProperty(name, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);

                if (p != null)
                {
                    return p.GetValue(source, null);
                }

                type = type.BaseType;
            }

            return null;
        }

        private static object GetValue_Imp(object source, string name, int index)
        {
#if UNITY_2021_2_OR_NEWER
            if (GetValue_Imp(source, name) is not IEnumerable enumerable)
            {
                return null;
            }
#else
            if (!(GetValue_Imp(source, name) is IEnumerable enumerable))
            {
                return null;
            }
#endif

            var enm = enumerable.GetEnumerator();

            for (var i = 0; i <= index; i++)
            {
                if (!enm.MoveNext())
                {
                    return null;
                }
            }

            return enm.Current;
        }
    }
}
#endif