EnumCaster.Benchmark.cs

    [MemoryDiagnoser]
    public class BM
    {
        enum E
        {
            a,
            b,
            c
        }

        E val;

        [GlobalSetup]
        public void Setup()
        {
            val = new Random().NextDouble() > .5 ? E.b : E.c;
        }

        [Benchmark]
        public long Cast() => Propagate((long)val);

        [Benchmark]
        public long IConvertible() => CastViaIConvertible(val);

        [Benchmark]
        public long EnumUtilsCast() => EnumUtils<E>.Cast(val);

        static long CastViaIConvertible<T>(T val) where T : struct, IConvertible
        {
            return val.ToInt64(null);
        }

        static long Propagate<T>(T val) where T : IConvertible
        {
            return val.ToInt64(null);
        }
    }

EnumCaster.cs

    public static class EnumUtils<T> where T : unmanaged, Enum
    {
        static readonly int _sxshift;

        static EnumUtils()
        {
            switch (default(T).GetTypeCode())
            {
                case TypeCode.Boolean:
                case TypeCode.Char:
                case TypeCode.Byte:
                case TypeCode.UInt16:
                case TypeCode.UInt32:
                case TypeCode.UInt64:
                case TypeCode.Int64:
                    _sxshift = 0;
                    break;

                case TypeCode.SByte:
                    _sxshift = (sizeof(long) - sizeof(sbyte)) * 8;
                    break;
                case TypeCode.Int16:
                    _sxshift = (sizeof(long) - sizeof(short)) * 8;
                    break;
                case TypeCode.Int32:
                    _sxshift = (sizeof(long) - sizeof(int)) * 8;
                    break;
                default:
                    throw new NotSupportedException();
            }
        }

        public static long Cast(T value)
        {
            long memory = 0;
            Unsafe.As<long, T>(ref memory) = value;
            return (memory << _sxshift) >> _sxshift;
        }
    }

EnumCaster.Tests.cs

    [TestFixture]
    public class Tests
    {
        enum EByte : byte
        {
        }

        enum ESByte : sbyte
        {
        }

        enum EUInt16 : ushort
        {
        }

        enum EInt16 : short
        {
        }

        enum EUInt32 : uint
        {
        }

        enum EInt32 : int
        {
        }

        //enum EUInt64 : ulong
        //{
        //}

        enum EInt64 : long
        {
        }

        [TestCase(typeof(EByte))]
        [TestCase(typeof(ESByte))]
        [TestCase(typeof(EUInt16))]
        [TestCase(typeof(EInt16))]
        [TestCase(typeof(EUInt32))]
        [TestCase(typeof(EInt32))]
        //[TestCase(typeof(EUInt64))]
        [TestCase(typeof(EInt64))]
        public void Test(Type enumType)
        {
            MethodInfo method = typeof(EnumUtils<>).MakeGenericType(enumType).GetMethod(nameof(EnumUtils<EByte>.Cast));

            foreach (long point in GetTestPoint())
            {
                foreach (long delta in GetDeltas())
                {
                    long value = point + delta;

                    IConvertible enumValue = (IConvertible)Enum.ToObject(enumType, value);
                    Assert.AreEqual(enumValue.ToInt64(null), method.Invoke(null, new object[] { enumValue }));
                }
            }
        }

        IEnumerable<long> GetDeltas()
        {
            for (int i = 0; i < 4; i++)
            {
                yield return i;
                yield return -i;
            }
        }

        IEnumerable<long> GetTestPoint()
        {
            yield return 0;

            for (int i = 1; i <= 4; i++)
            {
                yield return 1 << (8 * i);
                yield return 1 << (8 * i) - 1;
                yield return 1 << -((8 * i));
                yield return 1 << -((8 * i) - 1);
            }

            yield return long.MaxValue;
            yield return long.MinValue;
        }
    }